more sleep yield testing

build.c

@@ -485,6 +485,7 @@ void OnBuild(StringList cli_args)
                                   "-Wno-c99-extensions -Wno-c++98-compat-pedantic -Wno-c++98-compat "
                                   "-Wno-switch-enum -Wno-switch-default "
                                   "-Wno-reserved-identifier -Wno-reserved-macro-identifier "
+                                  "-Wno-missing-designated-field-initializers "
                                   "-Wno-unsafe-buffer-usage "
                                   "-Wno-c11-extensions -Wno-gnu-anonymous-struct -Wno-nested-anon-types ");
 

src/math.h

@@ -400,7 +400,7 @@ INLINE f32 math_sqrt(f32 x)
     return ix_sqrt_f32(x);
 }
 
-INLINE f32 math_sqrt64(f32 x)
+INLINE f64 math_sqrt64(f64 x)
 {
     return ix_sqrt_f64(x);
 }

src/prof_tracy.h

@@ -7,10 +7,10 @@
 
 #if PROFILING
 
-#define PROFILING_SYSTEM_TRACE 1
+#define PROFILING_SYSTEM_TRACE 0
 #define PROFILING_CAPTURE_FRAME_IMAGE 0
 #define PROFILING_LOCKS 0
-#define PROFILING_D3D 1
+#define PROFILING_D3D 0
 #define PROFILING_FILE_WSTR L".tracy"
 #define PROFILING_CMD_WSTR L"cmd /C start \"\" /wait tracy-capture.exe -o .tracy -a 127.0.0.1 && start \"\" tracy-profiler.exe .tracy"
 //#define PROFILING_CMD_WSTR L"tracy-profiler.exe -a 127.0.0.1"

src/snc.c

@@ -4,7 +4,8 @@
 #include "memory.h"
 #include "intrinsics.h"
 
-#define DEFAULT_MUTEX_SPIN 4000
+//#define DEFAULT_MUTEX_SPIN 4000
+#define DEFAULT_MUTEX_SPIN 0
 
 /* ========================== *
  * Mutex

src/sys.h

@@ -430,7 +430,11 @@ b32 sys_run_command(struct string cmd);
 
 
 
+/* ========================== *
+ * Scheduler
+ * ========================== */
 
+i64 sys_current_scheduler_period_ns(void);
 
 /* ========================== *
  * Wait

src/sys_win32.c

@@ -104,13 +104,6 @@ struct win32_window {
 #define NUM_WAIT_ADDR_BINS 65536
 #define NUM_WAIT_TIME_BINS 1024
 
-/* Defines the resolution of the scheduler interval.
- *
- * NOTE: This is not the actual rate that the scheduler runs at, just the
- * minimum amount of time that it can refer to. Smaller values mean that the
- * scheduler has to process a greater number of wait lists upon waking up. */
-#define SCHEDULER_MIN_INTERVAL_NS (KIBI(256))  /* ~262 microseconds */
-
 struct alignas(64) wait_list {
     /* =================================================== */
     u64 value;                                  /* 08 bytes */
@@ -146,8 +139,9 @@ STATIC_ASSERT(sizeof(struct wait_bin) == 64);  /* Padding validation (increase i
 STATIC_ASSERT(alignof(struct wait_bin) == 64);  /* Avoid false sharing */
 
 
-
+/* Assume scheduler cycle is 20hz at start to be conservative */
-
+#define DEFAULT_SCHEDULER_CYCLE_PERIOD_NS 50000000
+#define NUM_ROLLING_SCHEDULER_PERIODS 1000
 
 
 #define FIBER_NAME_PREFIX_CSTR "Fiber ["
@@ -269,9 +263,7 @@ struct alignas(64) job_queue {
 GLOBAL struct {
     SYSTEM_INFO info;
     i64 timer_start_qpc;
-    i64 qpc_per_second;
     i64 ns_per_qpc;
-    i32 scheduler_period_ms;
     u32 main_thread_id;
 
     wchar_t cmdline_args_wstr[8192];
@@ -307,7 +299,8 @@ GLOBAL struct {
 
 
     /* Scheduler */
-    struct atomic_i64 last_scheduler_interval;  /* TODO: Prevent false sharing */
+    struct atomic_i64 current_scheduler_cycle;  /* TODO: Prevent false sharing */
+    struct atomic_i64 current_scheduler_cycle_period_ns;  /* TODO: Prevent false sharing */
 
     /* Wait lists */
     struct atomic_u64 waiter_wake_gen; /* TODO: Prevent false sharing */
@@ -354,7 +347,14 @@ INTERNAL enum sys_priority job_priority_from_queue_kind(enum job_queue_kind queu
 
 
 
+/* ========================== *
+ * Scheduler
+ * ========================== */
 
+i64 sys_current_scheduler_period_ns(void)
+{
+    return atomic_i64_fetch(&G.current_scheduler_cycle_period_ns);
+}
 
 
 
@@ -980,7 +980,9 @@ INTERNAL SYS_THREAD_DEF(job_worker_entry, worker_ctx_arg)
                             i64 wait_timeout_ns = yield.wait.timeout_ns;
                             i64 wait_time = 0;
                             if (wait_timeout_ns > 0 && wait_timeout_ns < I64_MAX) {
-                                wait_time = (sys_time_ns() + wait_timeout_ns) / SCHEDULER_MIN_INTERVAL_NS - 1;
+                                u64 current_scheduler_cycle = atomic_i64_fetch(&G.current_scheduler_cycle);
+                                i64 current_scheduler_cycle_period_ns = atomic_i64_fetch(&G.current_scheduler_cycle_period_ns);
+                                wait_time = current_scheduler_cycle + max_i64((i64)((f64)wait_timeout_ns / (f64)current_scheduler_cycle_period_ns), 1);
                             }
 
                             u64 wait_addr_bin_index = (u64)wait_addr % NUM_WAIT_ADDR_BINS;
@@ -1003,7 +1005,7 @@ INTERNAL SYS_THREAD_DEF(job_worker_entry, worker_ctx_arg)
                                         }
                                     }
                                     if (wait_time != 0 && !cancel_wait) {
-                                        cancel_wait = wait_time <= atomic_i64_fetch(&G.last_scheduler_interval);
+                                        cancel_wait = wait_time <= atomic_i64_fetch(&G.current_scheduler_cycle);
                                     }
                                     if (!cancel_wait) {
                                         if (wait_addr != 0) {
@@ -1136,6 +1138,9 @@ INTERNAL SYS_THREAD_DEF(job_worker_entry, worker_ctx_arg)
 
 INTERNAL SYS_THREAD_DEF(job_scheduler_entry, _)
 {
+#if 0
+    timeBeginPeriod(1);
+
     (UNUSED)_;
     {
         i32 priority = THREAD_PRIORITY_TIME_CRITICAL;
@@ -1146,30 +1151,44 @@ INTERNAL SYS_THREAD_DEF(job_scheduler_entry, _)
 
     struct arena_temp scratch = scratch_begin_no_conflict();
 
-    HANDLE timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
+    /* Create ring buffer of scheduler cycles initialized to default value */
-    if (!timer) {
+    i32 periods_index = 0;
-        sys_panic(LIT("Failed to create high resolution timer"));
+    i64 periods[NUM_ROLLING_SCHEDULER_PERIODS] = ZI;
+    for (i32 i = 0; i < (i32)countof(periods); ++i) {
+        periods[i] = DEFAULT_SCHEDULER_CYCLE_PERIOD_NS;
     }
 
-    i64 last_interval = sys_time_ns() / SCHEDULER_MIN_INTERVAL_NS;
+    i64 last_cycle_ns = 0;
     while (atomic_i64_fetch(&G.workers_wake_gen) >= 0) {
+        __profn("Job scheduler cycle");
         {
-            __profn("Job scheduler wait");
+            __profn("Job scheduler sleep");
-            LARGE_INTEGER due = ZI;
+            Sleep(1);
-            //due.QuadPart = -(SCHEDULER_MIN_INTERVAL_NS / 100);
-            due.QuadPart = 0;
-            SetWaitableTimerEx(timer, &due, 0, NULL, NULL, NULL, 0);
-            WaitForSingleObject(timer, INFINITE);
         }
 
+        i64 now_ns = sys_time_ns();
+        if (last_cycle_ns != 0) {
+            i64 new_period_ns = now_ns - last_cycle_ns;
+            periods[periods_index++] = new_period_ns;
+            if (periods_index == countof(periods)) {
+                periods_index = 0;
+            }
+            /* Calculate mean period */
+            f64 periods_sum_ns = 0;
+            for (i32 i = 0; i < (i32)countof(periods); ++i) {
+                periods_sum_ns += (f64)periods[i];
+            }
+            f64 mean_ns = periods_sum_ns / (f64)countof(periods);
+            atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, math_round_to_int64(mean_ns));
+        }
+        last_cycle_ns = now_ns;
+
         u64 wake_gen = atomic_u64_fetch_add_u64(&G.waiter_wake_gen, 1);
-        i64 new_interval = sys_time_ns() / SCHEDULER_MIN_INTERVAL_NS;
+        i64 current_cycle = atomic_i64_fetch_add(&G.current_scheduler_cycle, 1) + 1;
-        atomic_i64_fetch_set(&G.last_scheduler_interval, new_interval);
         {
             __profn("Job scheduler run");
             struct arena_temp temp = arena_temp_begin(scratch.arena);
-            for (i64 interval = last_interval; interval < new_interval; ++interval) {
+            u64 wait_time_bin_index = (u64)current_cycle % NUM_WAIT_TIME_BINS;
-                u64 wait_time_bin_index = (u64)interval % NUM_WAIT_TIME_BINS;
             struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index];
             struct wait_list *wait_time_list = NULL;
 
@@ -1181,7 +1200,7 @@ INTERNAL SYS_THREAD_DEF(job_scheduler_entry, _)
                 {
                     /* Search for wait time list */
                     for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) {
-                            if (tmp->value == (u64)interval) {
+                        if (tmp->value == (u64)current_cycle) {
                             wait_time_list = tmp;
                         }
                     }
@@ -1346,14 +1365,511 @@ INTERNAL SYS_THREAD_DEF(job_scheduler_entry, _)
                 }
                 snc_unlock(&lock);
             }
-            }
-
             arena_temp_end(temp);
         }
-        last_interval = new_interval;
     }
 
     scratch_end(scratch);
+#elif 0
+    (UNUSED)_;
+    {
+        i32 priority = THREAD_PRIORITY_TIME_CRITICAL;
+        b32 success = SetThreadPriority(GetCurrentThread(), priority);
+        (UNUSED)success;
+        ASSERT(success);
+    }
+
+    struct arena_temp scratch = scratch_begin_no_conflict();
+
+    /* Create high resolution timer */
+    HANDLE timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
+    if (!timer) {
+        sys_panic(LIT("Failed to create high resolution timer"));
+    }
+
+    /* Set high resolution timer period */
+    {
+        LARGE_INTEGER due = ZI;
+        due.QuadPart = -1;
+        //i32 interval = 5;
+        //i32 interval = 1;
+        i32 interval = 1;
+        b32 success = false;
+        while (!success && interval <= 50) {
+            success = SetWaitableTimerEx(timer, &due, interval, NULL, NULL, NULL, 0);
+            ++interval;
+        }
+        if (!success) {
+            sys_panic(LIT("Failed to set scheduler timing period"));
+        }
+    }
+
+    /* Create ring buffer of scheduler cycles initialized to default value */
+    i32 periods_index = 0;
+    i64 periods[NUM_ROLLING_SCHEDULER_PERIODS] = ZI;
+    for (i32 i = 0; i < (i32)countof(periods); ++i) {
+        periods[i] = DEFAULT_SCHEDULER_CYCLE_PERIOD_NS;
+    }
+
+    i64 last_cycle_ns = 0;
+    while (atomic_i64_fetch(&G.workers_wake_gen) >= 0) {
+        __profn("Job scheduler cycle");
+        {
+            __profn("Job scheduler wait");
+            WaitForSingleObject(timer, INFINITE);
+        }
+
+        i64 now_ns = sys_time_ns();
+        if (last_cycle_ns != 0) {
+            i64 new_period_ns = now_ns - last_cycle_ns;
+            periods[periods_index++] = new_period_ns;
+            if (periods_index == countof(periods)) {
+                periods_index = 0;
+            }
+            /* Calculate mean period */
+            f64 periods_sum_ns = 0;
+            for (i32 i = 0; i < (i32)countof(periods); ++i) {
+                periods_sum_ns += (f64)periods[i];
+            }
+            f64 mean_ns = periods_sum_ns / (f64)countof(periods);
+            atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, math_round_to_int64(mean_ns));
+        }
+        last_cycle_ns = now_ns;
+
+        u64 wake_gen = atomic_u64_fetch_add_u64(&G.waiter_wake_gen, 1);
+        i64 current_cycle = atomic_i64_fetch_add(&G.current_scheduler_cycle, 1) + 1;
+        {
+            __profn("Job scheduler run");
+            struct arena_temp temp = arena_temp_begin(scratch.arena);
+            u64 wait_time_bin_index = (u64)current_cycle % NUM_WAIT_TIME_BINS;
+            struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index];
+            struct wait_list *wait_time_list = NULL;
+
+            /* Build list of waiters to resume */
+            i32 num_waiters = 0;
+            struct fiber **waiters = NULL;
+            {
+                while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause();
+                {
+                    /* Search for wait time list */
+                    for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) {
+                        if (tmp->value == (u64)current_cycle) {
+                            wait_time_list = tmp;
+                        }
+                    }
+
+                    if (wait_time_list) {
+                        /* Set waiter wake status & build waiters list */
+                        waiters = arena_push_array_no_zero(temp.arena, struct fiber *, wait_time_list->num_waiters);
+                        for (struct fiber *waiter = fiber_from_id(wait_time_list->first_waiter); waiter; waiter = fiber_from_id(waiter->next_time_waiter)) {
+                            if (atomic_u64_fetch_test_set(&waiter->wake_gen, 0, wake_gen) == 0) {
+                                waiters[num_waiters] = waiter;
+                                ++num_waiters;
+                            }
+
+                        }
+                    }
+                }
+                atomic_i32_fetch_set(&wait_time_bin->lock, 0);
+            }
+
+            /* Update wait lists */
+            for (i32 i = 0; i < num_waiters; ++i) {
+                struct fiber *waiter = waiters[i];
+                u64 wait_addr = waiter->wait_addr;
+                u64 wait_addr_bin_index = wait_addr % NUM_WAIT_ADDR_BINS;
+                struct wait_bin *wait_addr_bin = &G.wait_addr_bins[wait_addr_bin_index];
+
+                if (wait_addr != 0) while (atomic_i32_fetch_test_set(&wait_addr_bin->lock, 0, 1) != 0) ix_pause();
+                {
+                    /* Search for wait addr list */
+                    struct wait_list *wait_addr_list = NULL;
+                    if (wait_addr != 0) {
+                        for (struct wait_list *tmp = wait_addr_bin->first_wait_list; tmp && !wait_addr_list; tmp = tmp->next_in_bin) {
+                            if (tmp->value == (u64)wait_addr) {
+                                wait_addr_list = tmp;
+                            }
+                        }
+                    }
+
+                    while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause();
+                    {
+                        /* Remove from addr list */
+                        if (wait_addr_list) {
+                            if (--wait_addr_list->num_waiters == 0) {
+                                /* Free addr list */
+                                struct wait_list *prev = wait_addr_list->prev_in_bin;
+                                struct wait_list *next = wait_addr_list->next_in_bin;
+                                if (prev) {
+                                    prev->next_in_bin = next;
+                                } else {
+                                    wait_addr_bin->first_wait_list = next;
+                                }
+                                if (next) {
+                                    next->next_in_bin = prev;
+                                } else {
+                                    wait_addr_bin->last_wait_list = prev;
+                                }
+                                wait_addr_list->next_in_bin = wait_addr_bin->first_free_wait_list;
+                                wait_addr_bin->first_free_wait_list = wait_addr_list;
+                            } else {
+                                i16 prev_id = waiter->prev_addr_waiter;
+                                i16 next_id = waiter->next_addr_waiter;
+                                if (prev_id) {
+                                    fiber_from_id(prev_id)->next_addr_waiter = next_id;
+                                } else {
+                                    wait_addr_list->first_waiter = next_id;
+                                }
+                                if (next_id) {
+                                    fiber_from_id(next_id)->prev_addr_waiter = prev_id;
+                                } else {
+                                    wait_addr_list->last_waiter = prev_id;
+                                }
+                            }
+                            waiter->wait_addr = 0;
+                            waiter->prev_addr_waiter = 0;
+                            waiter->next_addr_waiter = 0;
+                        }
+                        /* Remove from time list */
+                        {
+                            if (--wait_time_list->num_waiters == 0) {
+                                /* Free time list */
+                                struct wait_list *prev = wait_time_list->prev_in_bin;
+                                struct wait_list *next = wait_time_list->next_in_bin;
+                                if (prev) {
+                                    prev->next_in_bin = next;
+                                } else {
+                                    wait_time_bin->first_wait_list = next;
+                                }
+                                if (next) {
+                                    next->next_in_bin = prev;
+                                } else {
+                                    wait_time_bin->last_wait_list = prev;
+                                }
+                                wait_time_list->next_in_bin = wait_time_bin->first_free_wait_list;
+                                wait_time_bin->first_free_wait_list = wait_time_list;
+                            } else {
+                                i16 prev_id = waiter->prev_time_waiter;
+                                i16 next_id = waiter->next_time_waiter;
+                                if (prev_id) {
+                                    fiber_from_id(prev_id)->next_time_waiter = next_id;
+                                } else {
+                                    wait_time_list->first_waiter = next_id;
+                                }
+                                if (next_id) {
+                                    fiber_from_id(next_id)->prev_time_waiter = prev_id;
+                                } else {
+                                    wait_time_list->last_waiter = prev_id;
+                                }
+                            }
+                            waiter->wait_time = 0;
+                            waiter->prev_time_waiter = 0;
+                            waiter->next_time_waiter = 0;
+                        }
+                    }
+                    atomic_i32_fetch_set(&wait_time_bin->lock, 0);
+                }
+                if (wait_addr != 0) atomic_i32_fetch_set(&wait_addr_bin->lock, 0);
+            }
+
+            /* Resume waiters */
+            /* TODO: Batch submit waiters based on queue kind rather than one at a time */
+            for (i32 i = 0; i < num_waiters; ++i) {
+                struct fiber *waiter = waiters[i];
+                enum job_queue_kind queue_kind = job_queue_kind_from_priority(waiter->job_priority);
+                struct job_queue *queue = &G.job_queues[queue_kind];
+                while (atomic_i32_fetch_test_set(&queue->lock, 0, 1) != 0) ix_pause();
+                {
+                    struct job_info *info = NULL;
+                    if (queue->first_free) {
+                        info = queue->first_free;
+                        queue->first_free = info->next;
+                    } else {
+                        info = arena_push_no_zero(queue->arena, struct job_info);
+                    }
+                    MEMZERO_STRUCT(info);
+                    info->count = 1;
+                    info->num_dispatched = waiter->job_id;
+                    info->func = waiter->job_func;
+                    info->sig = waiter->job_sig;
+                    info->counter = waiter->job_counter;
+                    info->fiber_id = waiter->id;
+                    if (queue->last) {
+                        queue->last->next = info;
+                    } else {
+                        queue->first = info;
+                    }
+                    queue->last = info;
+                    atomic_u64_fetch_set(&waiter->wake_gen, 0);
+                }
+                atomic_i32_fetch_set(&queue->lock, 0);
+            }
+
+            /* Wake workers */
+            /* TODO: Only wake necessary amount of workers */
+            if (num_waiters > 0) {
+                struct snc_lock lock = snc_lock_e(&G.workers_wake_mutex);
+                {
+                    atomic_i64_fetch_add(&G.num_jobs_in_queue, num_waiters);
+                    if (atomic_i64_fetch(&G.workers_wake_gen) >= 0) {
+                        atomic_i64_fetch_add(&G.workers_wake_gen, 1);
+                        snc_cv_broadcast(&G.workers_wake_cv);
+                    }
+                }
+                snc_unlock(&lock);
+            }
+            arena_temp_end(temp);
+        }
+    }
+
+    scratch_end(scratch);
+#else
+    (UNUSED)_;
+    {
+        i32 priority = THREAD_PRIORITY_TIME_CRITICAL;
+        b32 success = SetThreadPriority(GetCurrentThread(), priority);
+        (UNUSED)success;
+        ASSERT(success);
+
+        success = SetThreadAffinityMask(GetCurrentThread(), (1 << 14)) != 0;
+        ASSERT(success);
+    }
+
+    struct arena_temp scratch = scratch_begin_no_conflict();
+
+    /* Create high resolution timer */
+    HANDLE timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
+    if (!timer) {
+        sys_panic(LIT("Failed to create high resolution timer"));
+    }
+
+    /* Create ring buffer of scheduler cycles initialized to default value */
+    i32 periods_index = 0;
+    i64 periods[NUM_ROLLING_SCHEDULER_PERIODS] = ZI;
+    for (i32 i = 0; i < (i32)countof(periods); ++i) {
+        periods[i] = DEFAULT_SCHEDULER_CYCLE_PERIOD_NS;
+    }
+
+    i64 last_cycle_ns = 0;
+    while (atomic_i64_fetch(&G.workers_wake_gen) >= 0) {
+        __profn("Job scheduler cycle");
+        {
+            __profn("Job scheduler wait");
+            LARGE_INTEGER due = ZI;
+            due.QuadPart = -1;
+            //due.QuadPart = -10000;
+            //due.QuadPart = -32000;
+            //due.QuadPart = -12000;
+            //due.QuadPart = -8000;
+            SetWaitableTimerEx(timer, &due, 0, NULL, NULL, NULL, 0);
+            WaitForSingleObject(timer, INFINITE);
+        }
+
+        i64 now_ns = sys_time_ns();
+        if (last_cycle_ns != 0) {
+            i64 new_period_ns = now_ns - last_cycle_ns;
+            periods[periods_index++] = new_period_ns;
+            if (periods_index == countof(periods)) {
+                periods_index = 0;
+            }
+            /* Calculate mean period */
+            f64 periods_sum_ns = 0;
+            for (i32 i = 0; i < (i32)countof(periods); ++i) {
+                periods_sum_ns += (f64)periods[i];
+            }
+            f64 mean_ns = periods_sum_ns / (f64)countof(periods);
+            atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, math_round_to_int64(mean_ns));
+        }
+        last_cycle_ns = now_ns;
+
+        u64 wake_gen = atomic_u64_fetch_add_u64(&G.waiter_wake_gen, 1);
+        i64 current_cycle = atomic_i64_fetch_add(&G.current_scheduler_cycle, 1) + 1;
+        {
+            __profn("Job scheduler run");
+            struct arena_temp temp = arena_temp_begin(scratch.arena);
+            u64 wait_time_bin_index = (u64)current_cycle % NUM_WAIT_TIME_BINS;
+            struct wait_bin *wait_time_bin = &G.wait_time_bins[wait_time_bin_index];
+            struct wait_list *wait_time_list = NULL;
+
+            /* Build list of waiters to resume */
+            i32 num_waiters = 0;
+            struct fiber **waiters = NULL;
+            {
+                while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause();
+                {
+                    /* Search for wait time list */
+                    for (struct wait_list *tmp = wait_time_bin->first_wait_list; tmp && !wait_time_list; tmp = tmp->next_in_bin) {
+                        if (tmp->value == (u64)current_cycle) {
+                            wait_time_list = tmp;
+                        }
+                    }
+
+                    if (wait_time_list) {
+                        /* Set waiter wake status & build waiters list */
+                        waiters = arena_push_array_no_zero(temp.arena, struct fiber *, wait_time_list->num_waiters);
+                        for (struct fiber *waiter = fiber_from_id(wait_time_list->first_waiter); waiter; waiter = fiber_from_id(waiter->next_time_waiter)) {
+                            if (atomic_u64_fetch_test_set(&waiter->wake_gen, 0, wake_gen) == 0) {
+                                waiters[num_waiters] = waiter;
+                                ++num_waiters;
+                            }
+
+                        }
+                    }
+                }
+                atomic_i32_fetch_set(&wait_time_bin->lock, 0);
+            }
+
+            /* Update wait lists */
+            for (i32 i = 0; i < num_waiters; ++i) {
+                struct fiber *waiter = waiters[i];
+                u64 wait_addr = waiter->wait_addr;
+                u64 wait_addr_bin_index = wait_addr % NUM_WAIT_ADDR_BINS;
+                struct wait_bin *wait_addr_bin = &G.wait_addr_bins[wait_addr_bin_index];
+
+                if (wait_addr != 0) while (atomic_i32_fetch_test_set(&wait_addr_bin->lock, 0, 1) != 0) ix_pause();
+                {
+                    /* Search for wait addr list */
+                    struct wait_list *wait_addr_list = NULL;
+                    if (wait_addr != 0) {
+                        for (struct wait_list *tmp = wait_addr_bin->first_wait_list; tmp && !wait_addr_list; tmp = tmp->next_in_bin) {
+                            if (tmp->value == (u64)wait_addr) {
+                                wait_addr_list = tmp;
+                            }
+                        }
+                    }
+
+                    while (atomic_i32_fetch_test_set(&wait_time_bin->lock, 0, 1) != 0) ix_pause();
+                    {
+                        /* Remove from addr list */
+                        if (wait_addr_list) {
+                            if (--wait_addr_list->num_waiters == 0) {
+                                /* Free addr list */
+                                struct wait_list *prev = wait_addr_list->prev_in_bin;
+                                struct wait_list *next = wait_addr_list->next_in_bin;
+                                if (prev) {
+                                    prev->next_in_bin = next;
+                                } else {
+                                    wait_addr_bin->first_wait_list = next;
+                                }
+                                if (next) {
+                                    next->next_in_bin = prev;
+                                } else {
+                                    wait_addr_bin->last_wait_list = prev;
+                                }
+                                wait_addr_list->next_in_bin = wait_addr_bin->first_free_wait_list;
+                                wait_addr_bin->first_free_wait_list = wait_addr_list;
+                            } else {
+                                i16 prev_id = waiter->prev_addr_waiter;
+                                i16 next_id = waiter->next_addr_waiter;
+                                if (prev_id) {
+                                    fiber_from_id(prev_id)->next_addr_waiter = next_id;
+                                } else {
+                                    wait_addr_list->first_waiter = next_id;
+                                }
+                                if (next_id) {
+                                    fiber_from_id(next_id)->prev_addr_waiter = prev_id;
+                                } else {
+                                    wait_addr_list->last_waiter = prev_id;
+                                }
+                            }
+                            waiter->wait_addr = 0;
+                            waiter->prev_addr_waiter = 0;
+                            waiter->next_addr_waiter = 0;
+                        }
+                        /* Remove from time list */
+                        {
+                            if (--wait_time_list->num_waiters == 0) {
+                                /* Free time list */
+                                struct wait_list *prev = wait_time_list->prev_in_bin;
+                                struct wait_list *next = wait_time_list->next_in_bin;
+                                if (prev) {
+                                    prev->next_in_bin = next;
+                                } else {
+                                    wait_time_bin->first_wait_list = next;
+                                }
+                                if (next) {
+                                    next->next_in_bin = prev;
+                                } else {
+                                    wait_time_bin->last_wait_list = prev;
+                                }
+                                wait_time_list->next_in_bin = wait_time_bin->first_free_wait_list;
+                                wait_time_bin->first_free_wait_list = wait_time_list;
+                            } else {
+                                i16 prev_id = waiter->prev_time_waiter;
+                                i16 next_id = waiter->next_time_waiter;
+                                if (prev_id) {
+                                    fiber_from_id(prev_id)->next_time_waiter = next_id;
+                                } else {
+                                    wait_time_list->first_waiter = next_id;
+                                }
+                                if (next_id) {
+                                    fiber_from_id(next_id)->prev_time_waiter = prev_id;
+                                } else {
+                                    wait_time_list->last_waiter = prev_id;
+                                }
+                            }
+                            waiter->wait_time = 0;
+                            waiter->prev_time_waiter = 0;
+                            waiter->next_time_waiter = 0;
+                        }
+                    }
+                    atomic_i32_fetch_set(&wait_time_bin->lock, 0);
+                }
+                if (wait_addr != 0) atomic_i32_fetch_set(&wait_addr_bin->lock, 0);
+            }
+
+            /* Resume waiters */
+            /* TODO: Batch submit waiters based on queue kind rather than one at a time */
+            for (i32 i = 0; i < num_waiters; ++i) {
+                struct fiber *waiter = waiters[i];
+                enum job_queue_kind queue_kind = job_queue_kind_from_priority(waiter->job_priority);
+                struct job_queue *queue = &G.job_queues[queue_kind];
+                while (atomic_i32_fetch_test_set(&queue->lock, 0, 1) != 0) ix_pause();
+                {
+                    struct job_info *info = NULL;
+                    if (queue->first_free) {
+                        info = queue->first_free;
+                        queue->first_free = info->next;
+                    } else {
+                        info = arena_push_no_zero(queue->arena, struct job_info);
+                    }
+                    MEMZERO_STRUCT(info);
+                    info->count = 1;
+                    info->num_dispatched = waiter->job_id;
+                    info->func = waiter->job_func;
+                    info->sig = waiter->job_sig;
+                    info->counter = waiter->job_counter;
+                    info->fiber_id = waiter->id;
+                    if (queue->last) {
+                        queue->last->next = info;
+                    } else {
+                        queue->first = info;
+                    }
+                    queue->last = info;
+                    atomic_u64_fetch_set(&waiter->wake_gen, 0);
+                }
+                atomic_i32_fetch_set(&queue->lock, 0);
+            }
+
+            /* Wake workers */
+            /* TODO: Only wake necessary amount of workers */
+            if (num_waiters > 0) {
+                struct snc_lock lock = snc_lock_e(&G.workers_wake_mutex);
+                {
+                    atomic_i64_fetch_add(&G.num_jobs_in_queue, num_waiters);
+                    if (atomic_i64_fetch(&G.workers_wake_gen) >= 0) {
+                        atomic_i64_fetch_add(&G.workers_wake_gen, 1);
+                        snc_cv_broadcast(&G.workers_wake_cv);
+                    }
+                }
+                snc_unlock(&lock);
+            }
+            arena_temp_end(temp);
+        }
+    }
+
+    scratch_end(scratch);
+#endif
 }
 
 /* ========================== *
@@ -1366,8 +1882,8 @@ INTERNAL SYS_THREAD_DEF(test_entry, _)
     (UNUSED)_;
 
     /* Start scheduler */
+    atomic_i64_fetch_set(&G.current_scheduler_cycle_period_ns, DEFAULT_SCHEDULER_CYCLE_PERIOD_NS);
     struct sys_thread *scheduler_thread = sys_thread_alloc(job_scheduler_entry, NULL, LIT("Scheduler thread"), PROF_THREAD_GROUP_SCHEDULER);
-    while (atomic_i64_fetch(&G.last_scheduler_interval) == 0) ix_pause();
 
     /* Start workers */
     //G.num_worker_threads = 1;
@@ -3215,6 +3731,7 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
     (UNUSED)cmdline_wstr;
     (UNUSED)show_code;
 
+
 #if PROFILING
     {
         __profn("Launch profiler");
@@ -3233,6 +3750,21 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
 
     __profthread("Main thread", PROF_THREAD_GROUP_MAIN);
 
+    const wchar_t *error_msg = NULL;
+
+    /* Init timer */
+    {
+        LARGE_INTEGER qpf;
+        QueryPerformanceFrequency(&qpf);
+        G.ns_per_qpc = 1000000000 / qpf.QuadPart;
+    }
+    {
+        LARGE_INTEGER qpc;
+        QueryPerformanceCounter(&qpc);
+        G.timer_start_qpc = qpc.QuadPart;
+    }
+
+
     /* Init wait lists */
     G.wait_lists_arena = arena_alloc(GIBI(64));
 
@@ -3254,11 +3786,6 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
     MEMCPY(G.cmdline_args_wstr, cmdline_wstr, cmdline_len * sizeof(*cmdline_wstr));
     G.cmdline_args_wstr[cmdline_len] = 0;
 
-    const wchar_t *error_msg = NULL;
-
-    /* ========================== *
-     * Win32 setup
-     * ========================== */
 
     G.main_thread_id = GetCurrentThreadId();
     SetThreadDescription(GetCurrentThread(), L"Main thread");
@@ -3269,22 +3796,6 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
     /* Query system info */
     GetSystemInfo(&G.info);
 
-    LARGE_INTEGER qpf;
-    QueryPerformanceFrequency(&qpf);
-    G.qpc_per_second = qpf.QuadPart;
-    G.ns_per_qpc = 1000000000 / qpf.QuadPart;
-
-    LARGE_INTEGER qpc;
-    QueryPerformanceCounter(&qpc);
-    G.timer_start_qpc = qpc.QuadPart;
-
-    TIMECAPS caps;
-    timeGetDevCaps(&caps, sizeof(caps));
-    G.scheduler_period_ms = (i32)caps.wPeriodMin;
-
-    /* Set up timing period */
-    timeBeginPeriod(G.scheduler_period_ms);
-
     /* Set up threads */
     G.threads_arena = arena_alloc(GIBI(64));
 

src/user.c

@@ -91,7 +91,6 @@ GLOBAL struct {
     i32 console_log_color_indices[LOG_LEVEL_COUNT];
     f32 console_logs_height;
     b32 debug_console;
-    b32 profiler_launched;
 
     /* Window -> user */
     struct snc_mutex sys_events_mutex;

src/util.h

@@ -265,8 +265,9 @@ INLINE void sleep_precise(i64 sleep_time_ns)
 {
     __prof;
 
-    i64 tolerance = 200000;
+    i64 big_sleep = sys_current_scheduler_period_ns();
-    i64 big_sleep = 500000;
+    i64 tolerance = big_sleep * 0.5;
+    //i64 tolerance = 1000000000;
 
     i64 now_ns = sys_time_ns();
     i64 target_ns = now_ns + sleep_time_ns;