diff --git a/build.c b/build.c index 31ebbf4a..017de85c 100644 --- a/build.c +++ b/build.c @@ -485,6 +485,7 @@ void OnBuild(StringList cli_args) "-Wno-c99-extensions -Wno-c++98-compat-pedantic -Wno-c++98-compat " "-Wno-switch-enum -Wno-switch-default " "-Wno-reserved-identifier -Wno-reserved-macro-identifier " + "-Wno-missing-designated-field-initializers " "-Wno-unsafe-buffer-usage " "-Wno-c11-extensions -Wno-gnu-anonymous-struct -Wno-nested-anon-types "); diff --git a/src/math.h b/src/math.h index 806b762b..ea3397b8 100644 --- a/src/math.h +++ b/src/math.h @@ -400,7 +400,7 @@ INLINE f32 math_sqrt(f32 x) return ix_sqrt_f32(x); } -INLINE f32 math_sqrt64(f32 x) +INLINE f64 math_sqrt64(f64 x) { return ix_sqrt_f64(x); } diff --git a/src/prof_tracy.h b/src/prof_tracy.h index 415e696d..df55c783 100644 --- a/src/prof_tracy.h +++ b/src/prof_tracy.h @@ -7,10 +7,10 @@ #if PROFILING -#define PROFILING_SYSTEM_TRACE 1 +#define PROFILING_SYSTEM_TRACE 0 #define PROFILING_CAPTURE_FRAME_IMAGE 0 #define PROFILING_LOCKS 0 -#define PROFILING_D3D 1 +#define PROFILING_D3D 0 #define PROFILING_FILE_WSTR L".tracy" #define PROFILING_CMD_WSTR L"cmd /C start \"\" /wait tracy-capture.exe -o .tracy -a 127.0.0.1 && start \"\" tracy-profiler.exe .tracy" //#define PROFILING_CMD_WSTR L"tracy-profiler.exe -a 127.0.0.1" diff --git a/src/snc.c b/src/snc.c index 7d4efd8d..76c1c7fb 100644 --- a/src/snc.c +++ b/src/snc.c @@ -4,7 +4,8 @@ #include "memory.h" #include "intrinsics.h" -#define DEFAULT_MUTEX_SPIN 4000 +//#define DEFAULT_MUTEX_SPIN 4000 +#define DEFAULT_MUTEX_SPIN 0 /* ========================== * * Mutex diff --git a/src/sys.h b/src/sys.h index 8ca5892e..77515b95 100644 --- a/src/sys.h +++ b/src/sys.h @@ -430,7 +430,11 @@ b32 sys_run_command(struct string cmd); +/* ========================== * + * Scheduler + * ========================== */ +i64 sys_current_scheduler_period_ns(void); /* ========================== * * Wait diff --git a/src/sys_win32.c b/src/sys_win32.c index 9d174471..d34a67cf 100644 --- a/src/sys_win32.c +++ b/src/sys_win32.c @@ -104,13 +104,6 @@ struct win32_window { #define NUM_WAIT_ADDR_BINS 65536 #define NUM_WAIT_TIME_BINS 1024 -/* Defines the resolution of the scheduler interval. - * - * NOTE: This is not the actual rate that the scheduler runs at, just the - * minimum amount of time that it can refer to. Smaller values mean that the - * scheduler has to process a greater number of wait lists upon waking up. */ -#define SCHEDULER_MIN_INTERVAL_NS (KIBI(256)) /* ~262 microseconds */ - struct alignas(64) wait_list { /* =================================================== */ u64 value; /* 08 bytes */ @@ -146,8 +139,9 @@ STATIC_ASSERT(sizeof(struct wait_bin) == 64); /* Padding validation (increase i STATIC_ASSERT(alignof(struct wait_bin) == 64); /* Avoid false sharing */ - - +/* Assume scheduler cycle is 20hz at start to be conservative */ +#define DEFAULT_SCHEDULER_CYCLE_PERIOD_NS 50000000 +#define NUM_ROLLING_SCHEDULER_PERIODS 1000 #define FIBER_NAME_PREFIX_CSTR "Fiber [" @@ -269,9 +263,7 @@ struct alignas(64) job_queue { GLOBAL struct { SYSTEM_INFO info; i64 timer_start_qpc; - i64 qpc_per_second; i64 ns_per_qpc; - i32 scheduler_period_ms; u32 main_thread_id; wchar_t cmdline_args_wstr[8192]; @@ -307,7 +299,8 @@ GLOBAL struct { /* Scheduler */ - struct atomic_i64 last_scheduler_interval; /* TODO: Prevent false sharing */ + struct atomic_i64 current_scheduler_cycle; /* TODO: Prevent false sharing */ + struct atomic_i64 current_scheduler_cycle_period_ns; /* TODO: Prevent false sharing */ /* Wait lists */ struct atomic_u64 waiter_wake_gen; /* TODO: Prevent false sharing */ @@ -354,7 +347,14 @@ INTERNAL enum sys_priority job_priority_from_queue_kind(enum job_queue_kind queu +/* ========================== * + * Scheduler + * ========================== */ +i64 sys_current_scheduler_period_ns(void) +{ + return atomic_i64_fetch(&G.current_scheduler_cycle_period_ns); +} @@ -980,7 +980,9 @@ INTERNAL SYS_THREAD_DEF(job_worker_entry, worker_ctx_arg) i64 wait_timeout_ns = yield.wait.timeout_ns; i64 wait_time = 0; if (wait_timeout_ns > 0 && wait_timeout_ns < I64_MAX) { - wait_time = (sys_time_ns() + wait_timeout_ns) / SCHEDULER_MIN_INTERVAL_NS - 1; + u64 current_scheduler_cycle = atomic_i64_fetch(&G.current_scheduler_cycle); + i64 current_scheduler_cycle_period_ns = atomic_i64_fetch(&G.current_scheduler_cycle_period_ns); + wait_time = current_scheduler_cycle + max_i64((i64)((f64)wait_timeout_ns / (f64)current_scheduler_cycle_period_ns), 1); } u64 wait_addr_bin_index = (u64)wait_addr % NUM_WAIT_ADDR_BINS; @@ -1003,7 +1005,7 @@ INTERNAL SYS_THREAD_DEF(job_worker_entry, worker_ctx_arg) } } if (wait_time != 0 && !cancel_wait) { - cancel_wait = wait_time <= atomic_i64_fetch(&G.last_scheduler_interval); + cancel_wait = wait_time <= atomic_i64_fetch(&G.current_scheduler_cycle); } if (!cancel_wait) { if (wait_addr != 0) { @@ -1136,6 +1138,9 @@ INTERNAL SYS_THREAD_DEF(job_worker_entry, worker_ctx_arg) INTERNAL SYS_THREAD_DEF(job_scheduler_entry, _) { +#if 0 + timeBeginPeriod(1); + (UNUSED)_; { i32 priority = THREAD_PRIORITY_TIME_CRITICAL; @@ -1146,214 +1151,725 @@ INTERNAL SYS_THREAD_DEF(job_scheduler_entry, _) struct arena_temp scratch = scratch_begin_no_conflict(); + /* Create ring buffer of scheduler cycles initialized to default value */ + i32 periods_index = 0; + i64 periods[NUM_ROLLING_SCHEDULER_PERIODS] = ZI; + for (i32 i = 0; i < (i32)countof(periods); ++i) { + periods[i] = DEFAULT_SCHEDULER_CYCLE_PERIOD_NS; + } + + i64 last_cycle_ns = 0; + while (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { + __profn("Job scheduler cycle"); + { + __profn("Job scheduler sleep"); + Sleep(1); + } + + i64 now_ns = sys_time_ns(); + if (last_cycle_ns != 0) { + i64 new_period_ns = now_ns - last_cycle_ns; + periods[periods_index++] = new_period_ns; + if (periods_index == countof(periods)) { + periods_index = 0; + } + /* Calculate mean period */ + f64 periods_sum_ns = 0; + for (i32 i = 0; i < (i32)countof(periods); ++i) { + periods_sum_ns += (f64)periods[i]; + } + f64 mean_ns = periods_sum_ns / (f64)countof(periods); + atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, math_round_to_int64(mean_ns)); + } + last_cycle_ns = now_ns; + + u64 wake_gen = atomic_u64_fetch_add_u64(&G.waiter_wake_gen, 1); + i64 current_cycle = atomic_i64_fetch_add(&G.current_scheduler_cycle, 1) + 1; + { + __profn("Job scheduler run"); + struct arena_temp temp = arena_temp_begin(scratch.arena); + u64 wait_time_bin_index = (u64)current_cycle % NUM_WAIT_TIME_BINS; + struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index]; + struct wait_list *wait_time_list = NULL; + + /* Build list of waiters to resume */ + i32 num_waiters = 0; + struct fiber **waiters = NULL; + { + while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Search for wait time list */ + for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) { + if (tmp->value == (u64)current_cycle) { + wait_time_list = tmp; + } + } + + if (wait_time_list) { + /* Set waiter wake status & build waiters list */ + waiters = arena_push_array_no_zero(temp.arena, struct fiber *, wait_time_list->num_waiters); + for (struct fiber *waiter = fiber_from_id(wait_time_list->first_waiter); waiter; waiter = fiber_from_id(waiter->next_time_waiter)) { + if (atomic_u64_fetch_test_set(&waiter->wake_gen, 0, wake_gen) == 0) { + waiters[num_waiters] = waiter; + ++num_waiters; + } + + } + } + } + atomic_i32_fetch_set(&wait_time_bin->lock, 0); + } + + /* Update wait lists */ + for (i32 i = 0; i < num_waiters; ++i) { + struct fiber *waiter = waiters[i]; + u64 wait_addr = waiter->wait_addr; + u64 wait_addr_bin_index = wait_addr % NUM_WAIT_ADDR_BINS; + struct wait_bin *wait_addr_bin = &G.wait_addr_bins[wait_addr_bin_index]; + + if (wait_addr != 0) while (atomic_i32_fetch_test_set(&wait_addr_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Search for wait addr list */ + struct wait_list *wait_addr_list = NULL; + if (wait_addr != 0) { + for (struct wait_list *tmp = wait_addr_bin->first_wait_list; tmp && !wait_addr_list; tmp = tmp->next_in_bin) { + if (tmp->value == (u64)wait_addr) { + wait_addr_list = tmp; + } + } + } + + while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Remove from addr list */ + if (wait_addr_list) { + if (--wait_addr_list->num_waiters == 0) { + /* Free addr list */ + struct wait_list *prev = wait_addr_list->prev_in_bin; + struct wait_list *next = wait_addr_list->next_in_bin; + if (prev) { + prev->next_in_bin = next; + } else { + wait_addr_bin->first_wait_list = next; + } + if (next) { + next->next_in_bin = prev; + } else { + wait_addr_bin->last_wait_list = prev; + } + wait_addr_list->next_in_bin = wait_addr_bin->first_free_wait_list; + wait_addr_bin->first_free_wait_list = wait_addr_list; + } else { + i16 prev_id = waiter->prev_addr_waiter; + i16 next_id = waiter->next_addr_waiter; + if (prev_id) { + fiber_from_id(prev_id)->next_addr_waiter = next_id; + } else { + wait_addr_list->first_waiter = next_id; + } + if (next_id) { + fiber_from_id(next_id)->prev_addr_waiter = prev_id; + } else { + wait_addr_list->last_waiter = prev_id; + } + } + waiter->wait_addr = 0; + waiter->prev_addr_waiter = 0; + waiter->next_addr_waiter = 0; + } + /* Remove from time list */ + { + if (--wait_time_list->num_waiters == 0) { + /* Free time list */ + struct wait_list *prev = wait_time_list->prev_in_bin; + struct wait_list *next = wait_time_list->next_in_bin; + if (prev) { + prev->next_in_bin = next; + } else { + wait_time_bin->first_wait_list = next; + } + if (next) { + next->next_in_bin = prev; + } else { + wait_time_bin->last_wait_list = prev; + } + wait_time_list->next_in_bin = wait_time_bin->first_free_wait_list; + wait_time_bin->first_free_wait_list = wait_time_list; + } else { + i16 prev_id = waiter->prev_time_waiter; + i16 next_id = waiter->next_time_waiter; + if (prev_id) { + fiber_from_id(prev_id)->next_time_waiter = next_id; + } else { + wait_time_list->first_waiter = next_id; + } + if (next_id) { + fiber_from_id(next_id)->prev_time_waiter = prev_id; + } else { + wait_time_list->last_waiter = prev_id; + } + } + waiter->wait_time = 0; + waiter->prev_time_waiter = 0; + waiter->next_time_waiter = 0; + } + } + atomic_i32_fetch_set(&wait_time_bin->lock, 0); + } + if (wait_addr != 0) atomic_i32_fetch_set(&wait_addr_bin->lock, 0); + } + + /* Resume waiters */ + /* TODO: Batch submit waiters based on queue kind rather than one at a time */ + for (i32 i = 0; i < num_waiters; ++i) { + struct fiber *waiter = waiters[i]; + enum job_queue_kind queue_kind = job_queue_kind_from_priority(waiter->job_priority); + struct job_queue *queue = &G.job_queues[queue_kind]; + while (atomic_i32_fetch_test_set(&queue->lock, 0, 1) != 0) ix_pause(); + { + struct job_info *info = NULL; + if (queue->first_free) { + info = queue->first_free; + queue->first_free = info->next; + } else { + info = arena_push_no_zero(queue->arena, struct job_info); + } + MEMZERO_STRUCT(info); + info->count = 1; + info->num_dispatched = waiter->job_id; + info->func = waiter->job_func; + info->sig = waiter->job_sig; + info->counter = waiter->job_counter; + info->fiber_id = waiter->id; + if (queue->last) { + queue->last->next = info; + } else { + queue->first = info; + } + queue->last = info; + atomic_u64_fetch_set(&waiter->wake_gen, 0); + } + atomic_i32_fetch_set(&queue->lock, 0); + } + + /* Wake workers */ + /* TODO: Only wake necessary amount of workers */ + if (num_waiters > 0) { + struct snc_lock lock = snc_lock_e(&G.workers_wake_mutex); + { + atomic_i64_fetch_add(&G.num_jobs_in_queue, num_waiters); + if (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { + atomic_i64_fetch_add(&G.workers_wake_gen, 1); + snc_cv_broadcast(&G.workers_wake_cv); + } + } + snc_unlock(&lock); + } + arena_temp_end(temp); + } + } + + scratch_end(scratch); +#elif 0 + (UNUSED)_; + { + i32 priority = THREAD_PRIORITY_TIME_CRITICAL; + b32 success = SetThreadPriority(GetCurrentThread(), priority); + (UNUSED)success; + ASSERT(success); + } + + struct arena_temp scratch = scratch_begin_no_conflict(); + + /* Create high resolution timer */ HANDLE timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); if (!timer) { sys_panic(LIT("Failed to create high resolution timer")); } - i64 last_interval = sys_time_ns() / SCHEDULER_MIN_INTERVAL_NS; + /* Set high resolution timer period */ + { + LARGE_INTEGER due = ZI; + due.QuadPart = -1; + //i32 interval = 5; + //i32 interval = 1; + i32 interval = 1; + b32 success = false; + while (!success && interval <= 50) { + success = SetWaitableTimerEx(timer, &due, interval, NULL, NULL, NULL, 0); + ++interval; + } + if (!success) { + sys_panic(LIT("Failed to set scheduler timing period")); + } + } + + /* Create ring buffer of scheduler cycles initialized to default value */ + i32 periods_index = 0; + i64 periods[NUM_ROLLING_SCHEDULER_PERIODS] = ZI; + for (i32 i = 0; i < (i32)countof(periods); ++i) { + periods[i] = DEFAULT_SCHEDULER_CYCLE_PERIOD_NS; + } + + i64 last_cycle_ns = 0; while (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { + __profn("Job scheduler cycle"); { __profn("Job scheduler wait"); - LARGE_INTEGER due = ZI; - //due.QuadPart = -(SCHEDULER_MIN_INTERVAL_NS / 100); - due.QuadPart = 0; - SetWaitableTimerEx(timer, &due, 0, NULL, NULL, NULL, 0); WaitForSingleObject(timer, INFINITE); } + i64 now_ns = sys_time_ns(); + if (last_cycle_ns != 0) { + i64 new_period_ns = now_ns - last_cycle_ns; + periods[periods_index++] = new_period_ns; + if (periods_index == countof(periods)) { + periods_index = 0; + } + /* Calculate mean period */ + f64 periods_sum_ns = 0; + for (i32 i = 0; i < (i32)countof(periods); ++i) { + periods_sum_ns += (f64)periods[i]; + } + f64 mean_ns = periods_sum_ns / (f64)countof(periods); + atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, math_round_to_int64(mean_ns)); + } + last_cycle_ns = now_ns; + u64 wake_gen = atomic_u64_fetch_add_u64(&G.waiter_wake_gen, 1); - i64 new_interval = sys_time_ns() / SCHEDULER_MIN_INTERVAL_NS; - atomic_i64_fetch_set(&G.last_scheduler_interval, new_interval); + i64 current_cycle = atomic_i64_fetch_add(&G.current_scheduler_cycle, 1) + 1; { __profn("Job scheduler run"); struct arena_temp temp = arena_temp_begin(scratch.arena); - for (i64 interval = last_interval; interval < new_interval; ++interval) { - u64 wait_time_bin_index = (u64)interval % NUM_WAIT_TIME_BINS; - struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index]; - struct wait_list *wait_time_list = NULL; + u64 wait_time_bin_index = (u64)current_cycle % NUM_WAIT_TIME_BINS; + struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index]; + struct wait_list *wait_time_list = NULL; - /* Build list of waiters to resume */ - i32 num_waiters = 0; - struct fiber **waiters = NULL; + /* Build list of waiters to resume */ + i32 num_waiters = 0; + struct fiber **waiters = NULL; + { + while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); { - while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); - { - /* Search for wait time list */ - for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) { - if (tmp->value == (u64)interval) { - wait_time_list = tmp; + /* Search for wait time list */ + for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) { + if (tmp->value == (u64)current_cycle) { + wait_time_list = tmp; + } + } + + if (wait_time_list) { + /* Set waiter wake status & build waiters list */ + waiters = arena_push_array_no_zero(temp.arena, struct fiber *, wait_time_list->num_waiters); + for (struct fiber *waiter = fiber_from_id(wait_time_list->first_waiter); waiter; waiter = fiber_from_id(waiter->next_time_waiter)) { + if (atomic_u64_fetch_test_set(&waiter->wake_gen, 0, wake_gen) == 0) { + waiters[num_waiters] = waiter; + ++num_waiters; + } + + } + } + } + atomic_i32_fetch_set(&wait_time_bin->lock, 0); + } + + /* Update wait lists */ + for (i32 i = 0; i < num_waiters; ++i) { + struct fiber *waiter = waiters[i]; + u64 wait_addr = waiter->wait_addr; + u64 wait_addr_bin_index = wait_addr % NUM_WAIT_ADDR_BINS; + struct wait_bin *wait_addr_bin = &G.wait_addr_bins[wait_addr_bin_index]; + + if (wait_addr != 0) while (atomic_i32_fetch_test_set(&wait_addr_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Search for wait addr list */ + struct wait_list *wait_addr_list = NULL; + if (wait_addr != 0) { + for (struct wait_list *tmp = wait_addr_bin->first_wait_list; tmp && !wait_addr_list; tmp = tmp->next_in_bin) { + if (tmp->value == (u64)wait_addr) { + wait_addr_list = tmp; } } + } - if (wait_time_list) { - /* Set waiter wake status & build waiters list */ - waiters = arena_push_array_no_zero(temp.arena, struct fiber *, wait_time_list->num_waiters); - for (struct fiber *waiter = fiber_from_id(wait_time_list->first_waiter); waiter; waiter = fiber_from_id(waiter->next_time_waiter)) { - if (atomic_u64_fetch_test_set(&waiter->wake_gen, 0, wake_gen) == 0) { - waiters[num_waiters] = waiter; - ++num_waiters; + while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Remove from addr list */ + if (wait_addr_list) { + if (--wait_addr_list->num_waiters == 0) { + /* Free addr list */ + struct wait_list *prev = wait_addr_list->prev_in_bin; + struct wait_list *next = wait_addr_list->next_in_bin; + if (prev) { + prev->next_in_bin = next; + } else { + wait_addr_bin->first_wait_list = next; + } + if (next) { + next->next_in_bin = prev; + } else { + wait_addr_bin->last_wait_list = prev; + } + wait_addr_list->next_in_bin = wait_addr_bin->first_free_wait_list; + wait_addr_bin->first_free_wait_list = wait_addr_list; + } else { + i16 prev_id = waiter->prev_addr_waiter; + i16 next_id = waiter->next_addr_waiter; + if (prev_id) { + fiber_from_id(prev_id)->next_addr_waiter = next_id; + } else { + wait_addr_list->first_waiter = next_id; + } + if (next_id) { + fiber_from_id(next_id)->prev_addr_waiter = prev_id; + } else { + wait_addr_list->last_waiter = prev_id; } - } + waiter->wait_addr = 0; + waiter->prev_addr_waiter = 0; + waiter->next_addr_waiter = 0; + } + /* Remove from time list */ + { + if (--wait_time_list->num_waiters == 0) { + /* Free time list */ + struct wait_list *prev = wait_time_list->prev_in_bin; + struct wait_list *next = wait_time_list->next_in_bin; + if (prev) { + prev->next_in_bin = next; + } else { + wait_time_bin->first_wait_list = next; + } + if (next) { + next->next_in_bin = prev; + } else { + wait_time_bin->last_wait_list = prev; + } + wait_time_list->next_in_bin = wait_time_bin->first_free_wait_list; + wait_time_bin->first_free_wait_list = wait_time_list; + } else { + i16 prev_id = waiter->prev_time_waiter; + i16 next_id = waiter->next_time_waiter; + if (prev_id) { + fiber_from_id(prev_id)->next_time_waiter = next_id; + } else { + wait_time_list->first_waiter = next_id; + } + if (next_id) { + fiber_from_id(next_id)->prev_time_waiter = prev_id; + } else { + wait_time_list->last_waiter = prev_id; + } + } + waiter->wait_time = 0; + waiter->prev_time_waiter = 0; + waiter->next_time_waiter = 0; } } atomic_i32_fetch_set(&wait_time_bin->lock, 0); } - - /* Update wait lists */ - for (i32 i = 0; i < num_waiters; ++i) { - struct fiber *waiter = waiters[i]; - u64 wait_addr = waiter->wait_addr; - u64 wait_addr_bin_index = wait_addr % NUM_WAIT_ADDR_BINS; - struct wait_bin *wait_addr_bin = &G.wait_addr_bins[wait_addr_bin_index]; - - if (wait_addr != 0) while (atomic_i32_fetch_test_set(&wait_addr_bin->lock, 0, 1) != 0) ix_pause(); - { - /* Search for wait addr list */ - struct wait_list *wait_addr_list = NULL; - if (wait_addr != 0) { - for (struct wait_list *tmp = wait_addr_bin->first_wait_list; tmp && !wait_addr_list; tmp = tmp->next_in_bin) { - if (tmp->value == (u64)wait_addr) { - wait_addr_list = tmp; - } - } - } - - while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); - { - /* Remove from addr list */ - if (wait_addr_list) { - if (--wait_addr_list->num_waiters == 0) { - /* Free addr list */ - struct wait_list *prev = wait_addr_list->prev_in_bin; - struct wait_list *next = wait_addr_list->next_in_bin; - if (prev) { - prev->next_in_bin = next; - } else { - wait_addr_bin->first_wait_list = next; - } - if (next) { - next->next_in_bin = prev; - } else { - wait_addr_bin->last_wait_list = prev; - } - wait_addr_list->next_in_bin = wait_addr_bin->first_free_wait_list; - wait_addr_bin->first_free_wait_list = wait_addr_list; - } else { - i16 prev_id = waiter->prev_addr_waiter; - i16 next_id = waiter->next_addr_waiter; - if (prev_id) { - fiber_from_id(prev_id)->next_addr_waiter = next_id; - } else { - wait_addr_list->first_waiter = next_id; - } - if (next_id) { - fiber_from_id(next_id)->prev_addr_waiter = prev_id; - } else { - wait_addr_list->last_waiter = prev_id; - } - } - waiter->wait_addr = 0; - waiter->prev_addr_waiter = 0; - waiter->next_addr_waiter = 0; - } - /* Remove from time list */ - { - if (--wait_time_list->num_waiters == 0) { - /* Free time list */ - struct wait_list *prev = wait_time_list->prev_in_bin; - struct wait_list *next = wait_time_list->next_in_bin; - if (prev) { - prev->next_in_bin = next; - } else { - wait_time_bin->first_wait_list = next; - } - if (next) { - next->next_in_bin = prev; - } else { - wait_time_bin->last_wait_list = prev; - } - wait_time_list->next_in_bin = wait_time_bin->first_free_wait_list; - wait_time_bin->first_free_wait_list = wait_time_list; - } else { - i16 prev_id = waiter->prev_time_waiter; - i16 next_id = waiter->next_time_waiter; - if (prev_id) { - fiber_from_id(prev_id)->next_time_waiter = next_id; - } else { - wait_time_list->first_waiter = next_id; - } - if (next_id) { - fiber_from_id(next_id)->prev_time_waiter = prev_id; - } else { - wait_time_list->last_waiter = prev_id; - } - } - waiter->wait_time = 0; - waiter->prev_time_waiter = 0; - waiter->next_time_waiter = 0; - } - } - atomic_i32_fetch_set(&wait_time_bin->lock, 0); - } - if (wait_addr != 0) atomic_i32_fetch_set(&wait_addr_bin->lock, 0); - } - - /* Resume waiters */ - /* TODO: Batch submit waiters based on queue kind rather than one at a time */ - for (i32 i = 0; i < num_waiters; ++i) { - struct fiber *waiter = waiters[i]; - enum job_queue_kind queue_kind = job_queue_kind_from_priority(waiter->job_priority); - struct job_queue *queue = &G.job_queues[queue_kind]; - while (atomic_i32_fetch_test_set(&queue->lock, 0, 1) != 0) ix_pause(); - { - struct job_info *info = NULL; - if (queue->first_free) { - info = queue->first_free; - queue->first_free = info->next; - } else { - info = arena_push_no_zero(queue->arena, struct job_info); - } - MEMZERO_STRUCT(info); - info->count = 1; - info->num_dispatched = waiter->job_id; - info->func = waiter->job_func; - info->sig = waiter->job_sig; - info->counter = waiter->job_counter; - info->fiber_id = waiter->id; - if (queue->last) { - queue->last->next = info; - } else { - queue->first = info; - } - queue->last = info; - atomic_u64_fetch_set(&waiter->wake_gen, 0); - } - atomic_i32_fetch_set(&queue->lock, 0); - } - - /* Wake workers */ - /* TODO: Only wake necessary amount of workers */ - if (num_waiters > 0) { - struct snc_lock lock = snc_lock_e(&G.workers_wake_mutex); - { - atomic_i64_fetch_add(&G.num_jobs_in_queue, num_waiters); - if (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { - atomic_i64_fetch_add(&G.workers_wake_gen, 1); - snc_cv_broadcast(&G.workers_wake_cv); - } - } - snc_unlock(&lock); - } + if (wait_addr != 0) atomic_i32_fetch_set(&wait_addr_bin->lock, 0); } + /* Resume waiters */ + /* TODO: Batch submit waiters based on queue kind rather than one at a time */ + for (i32 i = 0; i < num_waiters; ++i) { + struct fiber *waiter = waiters[i]; + enum job_queue_kind queue_kind = job_queue_kind_from_priority(waiter->job_priority); + struct job_queue *queue = &G.job_queues[queue_kind]; + while (atomic_i32_fetch_test_set(&queue->lock, 0, 1) != 0) ix_pause(); + { + struct job_info *info = NULL; + if (queue->first_free) { + info = queue->first_free; + queue->first_free = info->next; + } else { + info = arena_push_no_zero(queue->arena, struct job_info); + } + MEMZERO_STRUCT(info); + info->count = 1; + info->num_dispatched = waiter->job_id; + info->func = waiter->job_func; + info->sig = waiter->job_sig; + info->counter = waiter->job_counter; + info->fiber_id = waiter->id; + if (queue->last) { + queue->last->next = info; + } else { + queue->first = info; + } + queue->last = info; + atomic_u64_fetch_set(&waiter->wake_gen, 0); + } + atomic_i32_fetch_set(&queue->lock, 0); + } + + /* Wake workers */ + /* TODO: Only wake necessary amount of workers */ + if (num_waiters > 0) { + struct snc_lock lock = snc_lock_e(&G.workers_wake_mutex); + { + atomic_i64_fetch_add(&G.num_jobs_in_queue, num_waiters); + if (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { + atomic_i64_fetch_add(&G.workers_wake_gen, 1); + snc_cv_broadcast(&G.workers_wake_cv); + } + } + snc_unlock(&lock); + } arena_temp_end(temp); } - last_interval = new_interval; } scratch_end(scratch); +#else + (UNUSED)_; + { + i32 priority = THREAD_PRIORITY_TIME_CRITICAL; + b32 success = SetThreadPriority(GetCurrentThread(), priority); + (UNUSED)success; + ASSERT(success); + + success = SetThreadAffinityMask(GetCurrentThread(), (1 << 14)) != 0; + ASSERT(success); + } + + struct arena_temp scratch = scratch_begin_no_conflict(); + + /* Create high resolution timer */ + HANDLE timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + if (!timer) { + sys_panic(LIT("Failed to create high resolution timer")); + } + + /* Create ring buffer of scheduler cycles initialized to default value */ + i32 periods_index = 0; + i64 periods[NUM_ROLLING_SCHEDULER_PERIODS] = ZI; + for (i32 i = 0; i < (i32)countof(periods); ++i) { + periods[i] = DEFAULT_SCHEDULER_CYCLE_PERIOD_NS; + } + + i64 last_cycle_ns = 0; + while (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { + __profn("Job scheduler cycle"); + { + __profn("Job scheduler wait"); + LARGE_INTEGER due = ZI; + due.QuadPart = -1; + //due.QuadPart = -10000; + //due.QuadPart = -32000; + //due.QuadPart = -12000; + //due.QuadPart = -8000; + SetWaitableTimerEx(timer, &due, 0, NULL, NULL, NULL, 0); + WaitForSingleObject(timer, INFINITE); + } + + i64 now_ns = sys_time_ns(); + if (last_cycle_ns != 0) { + i64 new_period_ns = now_ns - last_cycle_ns; + periods[periods_index++] = new_period_ns; + if (periods_index == countof(periods)) { + periods_index = 0; + } + /* Calculate mean period */ + f64 periods_sum_ns = 0; + for (i32 i = 0; i < (i32)countof(periods); ++i) { + periods_sum_ns += (f64)periods[i]; + } + f64 mean_ns = periods_sum_ns / (f64)countof(periods); + atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, math_round_to_int64(mean_ns)); + } + last_cycle_ns = now_ns; + + u64 wake_gen = atomic_u64_fetch_add_u64(&G.waiter_wake_gen, 1); + i64 current_cycle = atomic_i64_fetch_add(&G.current_scheduler_cycle, 1) + 1; + { + __profn("Job scheduler run"); + struct arena_temp temp = arena_temp_begin(scratch.arena); + u64 wait_time_bin_index = (u64)current_cycle % NUM_WAIT_TIME_BINS; + struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index]; + struct wait_list *wait_time_list = NULL; + + /* Build list of waiters to resume */ + i32 num_waiters = 0; + struct fiber **waiters = NULL; + { + while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Search for wait time list */ + for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) { + if (tmp->value == (u64)current_cycle) { + wait_time_list = tmp; + } + } + + if (wait_time_list) { + /* Set waiter wake status & build waiters list */ + waiters = arena_push_array_no_zero(temp.arena, struct fiber *, wait_time_list->num_waiters); + for (struct fiber *waiter = fiber_from_id(wait_time_list->first_waiter); waiter; waiter = fiber_from_id(waiter->next_time_waiter)) { + if (atomic_u64_fetch_test_set(&waiter->wake_gen, 0, wake_gen) == 0) { + waiters[num_waiters] = waiter; + ++num_waiters; + } + + } + } + } + atomic_i32_fetch_set(&wait_time_bin->lock, 0); + } + + /* Update wait lists */ + for (i32 i = 0; i < num_waiters; ++i) { + struct fiber *waiter = waiters[i]; + u64 wait_addr = waiter->wait_addr; + u64 wait_addr_bin_index = wait_addr % NUM_WAIT_ADDR_BINS; + struct wait_bin *wait_addr_bin = &G.wait_addr_bins[wait_addr_bin_index]; + + if (wait_addr != 0) while (atomic_i32_fetch_test_set(&wait_addr_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Search for wait addr list */ + struct wait_list *wait_addr_list = NULL; + if (wait_addr != 0) { + for (struct wait_list *tmp = wait_addr_bin->first_wait_list; tmp && !wait_addr_list; tmp = tmp->next_in_bin) { + if (tmp->value == (u64)wait_addr) { + wait_addr_list = tmp; + } + } + } + + while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause(); + { + /* Remove from addr list */ + if (wait_addr_list) { + if (--wait_addr_list->num_waiters == 0) { + /* Free addr list */ + struct wait_list *prev = wait_addr_list->prev_in_bin; + struct wait_list *next = wait_addr_list->next_in_bin; + if (prev) { + prev->next_in_bin = next; + } else { + wait_addr_bin->first_wait_list = next; + } + if (next) { + next->next_in_bin = prev; + } else { + wait_addr_bin->last_wait_list = prev; + } + wait_addr_list->next_in_bin = wait_addr_bin->first_free_wait_list; + wait_addr_bin->first_free_wait_list = wait_addr_list; + } else { + i16 prev_id = waiter->prev_addr_waiter; + i16 next_id = waiter->next_addr_waiter; + if (prev_id) { + fiber_from_id(prev_id)->next_addr_waiter = next_id; + } else { + wait_addr_list->first_waiter = next_id; + } + if (next_id) { + fiber_from_id(next_id)->prev_addr_waiter = prev_id; + } else { + wait_addr_list->last_waiter = prev_id; + } + } + waiter->wait_addr = 0; + waiter->prev_addr_waiter = 0; + waiter->next_addr_waiter = 0; + } + /* Remove from time list */ + { + if (--wait_time_list->num_waiters == 0) { + /* Free time list */ + struct wait_list *prev = wait_time_list->prev_in_bin; + struct wait_list *next = wait_time_list->next_in_bin; + if (prev) { + prev->next_in_bin = next; + } else { + wait_time_bin->first_wait_list = next; + } + if (next) { + next->next_in_bin = prev; + } else { + wait_time_bin->last_wait_list = prev; + } + wait_time_list->next_in_bin = wait_time_bin->first_free_wait_list; + wait_time_bin->first_free_wait_list = wait_time_list; + } else { + i16 prev_id = waiter->prev_time_waiter; + i16 next_id = waiter->next_time_waiter; + if (prev_id) { + fiber_from_id(prev_id)->next_time_waiter = next_id; + } else { + wait_time_list->first_waiter = next_id; + } + if (next_id) { + fiber_from_id(next_id)->prev_time_waiter = prev_id; + } else { + wait_time_list->last_waiter = prev_id; + } + } + waiter->wait_time = 0; + waiter->prev_time_waiter = 0; + waiter->next_time_waiter = 0; + } + } + atomic_i32_fetch_set(&wait_time_bin->lock, 0); + } + if (wait_addr != 0) atomic_i32_fetch_set(&wait_addr_bin->lock, 0); + } + + /* Resume waiters */ + /* TODO: Batch submit waiters based on queue kind rather than one at a time */ + for (i32 i = 0; i < num_waiters; ++i) { + struct fiber *waiter = waiters[i]; + enum job_queue_kind queue_kind = job_queue_kind_from_priority(waiter->job_priority); + struct job_queue *queue = &G.job_queues[queue_kind]; + while (atomic_i32_fetch_test_set(&queue->lock, 0, 1) != 0) ix_pause(); + { + struct job_info *info = NULL; + if (queue->first_free) { + info = queue->first_free; + queue->first_free = info->next; + } else { + info = arena_push_no_zero(queue->arena, struct job_info); + } + MEMZERO_STRUCT(info); + info->count = 1; + info->num_dispatched = waiter->job_id; + info->func = waiter->job_func; + info->sig = waiter->job_sig; + info->counter = waiter->job_counter; + info->fiber_id = waiter->id; + if (queue->last) { + queue->last->next = info; + } else { + queue->first = info; + } + queue->last = info; + atomic_u64_fetch_set(&waiter->wake_gen, 0); + } + atomic_i32_fetch_set(&queue->lock, 0); + } + + /* Wake workers */ + /* TODO: Only wake necessary amount of workers */ + if (num_waiters > 0) { + struct snc_lock lock = snc_lock_e(&G.workers_wake_mutex); + { + atomic_i64_fetch_add(&G.num_jobs_in_queue, num_waiters); + if (atomic_i64_fetch(&G.workers_wake_gen) >= 0) { + atomic_i64_fetch_add(&G.workers_wake_gen, 1); + snc_cv_broadcast(&G.workers_wake_cv); + } + } + snc_unlock(&lock); + } + arena_temp_end(temp); + } + } + + scratch_end(scratch); +#endif } /* ========================== * @@ -1366,8 +1882,8 @@ INTERNAL SYS_THREAD_DEF(test_entry, _) (UNUSED)_; /* Start scheduler */ + atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, DEFAULT_SCHEDULER_CYCLE_PERIOD_NS); struct sys_thread *scheduler_thread = sys_thread_alloc(job_scheduler_entry, NULL, LIT("Scheduler thread"), PROF_THREAD_GROUP_SCHEDULER); - while (atomic_i64_fetch(&G.last_scheduler_interval) == 0) ix_pause(); /* Start workers */ //G.num_worker_threads = 1; @@ -3215,6 +3731,7 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, (UNUSED)cmdline_wstr; (UNUSED)show_code; + #if PROFILING { __profn("Launch profiler"); @@ -3233,6 +3750,21 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, __profthread("Main thread", PROF_THREAD_GROUP_MAIN); + const wchar_t *error_msg = NULL; + + /* Init timer */ + { + LARGE_INTEGER qpf; + QueryPerformanceFrequency(&qpf); + G.ns_per_qpc = 1000000000 / qpf.QuadPart; + } + { + LARGE_INTEGER qpc; + QueryPerformanceCounter(&qpc); + G.timer_start_qpc = qpc.QuadPart; + } + + /* Init wait lists */ G.wait_lists_arena = arena_alloc(GIBI(64)); @@ -3254,11 +3786,6 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, MEMCPY(G.cmdline_args_wstr, cmdline_wstr, cmdline_len * sizeof(*cmdline_wstr)); G.cmdline_args_wstr[cmdline_len] = 0; - const wchar_t *error_msg = NULL; - - /* ========================== * - * Win32 setup - * ========================== */ G.main_thread_id = GetCurrentThreadId(); SetThreadDescription(GetCurrentThread(), L"Main thread"); @@ -3269,22 +3796,6 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, /* Query system info */ GetSystemInfo(&G.info); - LARGE_INTEGER qpf; - QueryPerformanceFrequency(&qpf); - G.qpc_per_second = qpf.QuadPart; - G.ns_per_qpc = 1000000000 / qpf.QuadPart; - - LARGE_INTEGER qpc; - QueryPerformanceCounter(&qpc); - G.timer_start_qpc = qpc.QuadPart; - - TIMECAPS caps; - timeGetDevCaps(&caps, sizeof(caps)); - G.scheduler_period_ms = (i32)caps.wPeriodMin; - - /* Set up timing period */ - timeBeginPeriod(G.scheduler_period_ms); - /* Set up threads */ G.threads_arena = arena_alloc(GIBI(64)); diff --git a/src/user.c b/src/user.c index 11e0acc1..ff28d804 100644 --- a/src/user.c +++ b/src/user.c @@ -91,7 +91,6 @@ GLOBAL struct { i32 console_log_color_indices[LOG_LEVEL_COUNT]; f32 console_logs_height; b32 debug_console; - b32 profiler_launched; /* Window -> user */ struct snc_mutex sys_events_mutex; diff --git a/src/util.h b/src/util.h index 684fa4a6..efcee4fb 100644 --- a/src/util.h +++ b/src/util.h @@ -265,8 +265,9 @@ INLINE void sleep_precise(i64 sleep_time_ns) { __prof; - i64 tolerance = 200000; - i64 big_sleep = 500000; + i64 big_sleep = sys_current_scheduler_period_ns(); + i64 tolerance = big_sleep * 0.5; + //i64 tolerance = 1000000000; i64 now_ns = sys_time_ns(); i64 target_ns = now_ns + sleep_time_ns;