app startup as job

src/app.c

@@ -26,7 +26,6 @@
 GLOBAL struct {
     struct arena *arena;
     struct string write_path;
-    struct snc_counter exit_fence;
 } G = ZI, DEBUG_ALIAS(G, G_app);
 
 /* ========================== *
@@ -172,10 +171,10 @@ INTERNAL struct app_arg_list parse_args(struct arena *arena, struct string args_
  * Entry point
  * ========================== */
 
-void sys_app_entry(struct string args_str)
+void sys_app_startup(struct string args_str)
 {
+    __prof;
     struct arena_temp scratch = scratch_begin_no_conflict();
-    snc_counter_add(&G.exit_fence, 1);
 
     struct app_arg_list args = parse_args(scratch.arena, args_str);
     struct string logfile_name = LIT("log.log");
@@ -278,9 +277,11 @@ void sys_app_entry(struct string args_str)
     }
 #endif
 
-    /* Startup systems */
+    /* Global systems */
     resource_startup();
     gp_startup();
+
+    /* Subsystems */
     struct sock_startup_receipt sock_sr = sock_startup();
     struct host_startup_receipt host_sr = host_startup(&sock_sr);
     struct asset_cache_startup_receipt asset_cache_sr = asset_cache_startup();
@@ -291,15 +292,14 @@ void sys_app_entry(struct string args_str)
     struct sound_startup_receipt sound_sr = sound_startup(&asset_cache_sr);
     struct draw_startup_receipt draw_sr = draw_startup(&font_sr);
     struct sim_startup_receipt sim_sr = sim_startup();
+
+    /* Interface systems */
     struct playback_startup_receipt playback_sr = playback_startup(&mixer_sr);
     struct user_startup_receipt user_sr = user_startup(&font_sr, &sprite_sr, &draw_sr, &asset_cache_sr, &sound_sr, &mixer_sr, &host_sr, &sim_sr, connect_address);
 
     (UNUSED)user_sr;
     (UNUSED)playback_sr;
 
-    /* Wait for app_exit() */
-    snc_counter_wait(&G.exit_fence);
-
 #if 0
     /* Write window settings to file */
     {
@@ -326,11 +326,6 @@ void sys_app_entry(struct string args_str)
     sys_window_release(window);
 #endif
 
-    logf_info("Program exited normally");
+    //logf_info("Program exited normally");
     scratch_end(scratch);
 }
-
-void app_exit(void)
-{
-    snc_counter_add(&G.exit_fence, -1);
-}

src/app.h

@@ -3,6 +3,4 @@
 
 struct string app_write_path_cat(struct arena *arena, struct string filename);
 
-void app_exit(void);
-
 #endif

src/config.h

@@ -42,7 +42,7 @@
 #define SIM_TILES_PER_UNIT_SQRT         (4)
 #define SIM_TILES_PER_CHUNK_SQRT        (16)
 
-#define SIM_TICKS_PER_SECOND 100
+#define SIM_TICKS_PER_SECOND 50
 //#define SIM_TIMESCALE 1
 /* Like USER_INTERP_RATIO, but applies to snapshots received by the local sim from the
  * master sim (how far back in time should the client render the server's state) */

src/gp_dx12.c

@@ -326,10 +326,12 @@ GLOBAL struct {
     struct snc_mutex global_submit_mutex;
     struct command_queue *command_queues[DX12_NUM_QUEUES];
 
-    /* Evictor thread */
+    /* Evictor job */
-    struct atomic32 evictor_thread_shutdown;
+    struct snc_counter evictor_job_counter;
-    HANDLE evictor_thread_wake_event;
+    struct snc_cv evictor_wake_cv;
-    struct sys_thread *evictor_thread;
+    struct snc_mutex evictor_wake_mutex;
+    i64 evictor_wake_gen;
+    b32 evictor_shutdown;
 } G = ZI, DEBUG_ALIAS(G, G_gp_dx12);
 
 /* ========================== *
@@ -343,7 +345,7 @@ INTERNAL void dx12_init_pipelines(void);
 INTERNAL struct cpu_descriptor_heap *cpu_descriptor_heap_alloc(enum D3D12_DESCRIPTOR_HEAP_TYPE type);
 INTERNAL struct command_queue *command_queue_alloc(enum D3D12_COMMAND_LIST_TYPE type, enum D3D12_COMMAND_QUEUE_PRIORITY priority, struct string dbg_name);
 INTERNAL void command_queue_release(struct command_queue *cq);
-INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg);
+INTERNAL SYS_JOB_DEF(dx12_evictor_job, _);
 INTERNAL void fenced_release(void *data, enum fenced_release_kind kind);
 
 #if RESOURCE_RELOADING
@@ -390,9 +392,8 @@ void gp_startup(void)
 #endif
     sys_on_exit(gp_shutdown);
 
-    /* Start evictor thread */
+    /* Start evictor job */
-    G.evictor_thread_wake_event = CreateEvent(0, 0, 0, 0);
+    sys_run(1, dx12_evictor_job, 0, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &G.evictor_job_counter);
-    G.evictor_thread = sys_thread_alloc(evictor_thread_entry_point, 0, LIT("GPU resource evictor thread"), PROF_THREAD_GROUP_EVICTORS);
 }
 
 INTERNAL SYS_EXIT_FUNC(gp_shutdown)
@@ -410,9 +411,13 @@ INTERNAL SYS_EXIT_FUNC(gp_shutdown)
     (UNUSED)command_queue_release;
 #endif
 
-    atomic32_fetch_set(&G.evictor_thread_shutdown, 1);
+    {
-    SetEvent(G.evictor_thread_wake_event);
+        struct snc_lock lock = snc_lock_e(&G.evictor_wake_mutex);
-    sys_thread_wait_release(G.evictor_thread);
+        G.evictor_shutdown = 1;
+        snc_cv_signal(&G.evictor_wake_cv, I32_MAX);
+        snc_unlock(&lock);
+    }
+    snc_counter_wait(&G.evictor_job_counter);
 }
 
 /* ========================== *
@@ -723,7 +728,6 @@ INTERNAL HRESULT dx12_include_open(ID3DInclude *d3d_handler, D3D_INCLUDE_TYPE in
     struct resource *res = &handler->open_resources[handler->num_open_resources++];
     *res = resource_open(name);
     if (resource_exists(res)) {
-        ++handler->num_open_resources;
         struct string data = resource_get_data(res);
         *data_out = data.text;
         *data_len_out = data.len;
@@ -748,6 +752,7 @@ INTERNAL HRESULT dx12_include_close(ID3DInclude *d3d_handler, LPCVOID data)
 
 INTERNAL struct dx12_include_handler *dx12_include_handler_alloc(struct arena *arena, struct pipeline *pipeline)
 {
+    __prof;
     struct dx12_include_handler *handler = arena_push(arena, struct dx12_include_handler);
     handler->d3d_handler.lpVtbl = &handler->vtbl;
     handler->vtbl.Open = dx12_include_open;
@@ -758,6 +763,7 @@ INTERNAL struct dx12_include_handler *dx12_include_handler_alloc(struct arena *a
 
 INTERNAL void dx12_include_handler_release(struct dx12_include_handler *handler)
 {
+    __prof;
     for (u64 i = 0; i < handler->num_open_resources; ++i) {
         ASSERT(0); /* Resource should have been closed by handler by now */
         struct resource *res = &handler->open_resources[i];
@@ -1566,7 +1572,12 @@ INTERNAL void fenced_release(void *data, enum fenced_release_kind kind)
     }
 
     /* Wake evictor */
-    SetEvent(G.evictor_thread_wake_event);
+    struct snc_lock lock = snc_lock_e(&G.evictor_wake_mutex);
+    {
+        ++G.evictor_wake_gen;
+        snc_cv_signal(&G.evictor_wake_cv, I32_MAX);
+    }
+    snc_unlock(&lock);
 }
 
 /* ========================== *
@@ -2191,7 +2202,7 @@ INTERNAL D3D12_INDEX_BUFFER_VIEW ibv_from_command_buffer(struct command_buffer *
 }
 
 /* ========================== *
- * Texture
+ * Wait job
  * ========================== */
 
 struct dx12_wait_fence_job_sig {
@@ -2206,6 +2217,7 @@ INTERNAL SYS_JOB_DEF(dx12_wait_fence_job, job)
     ID3D12Fence *fence = sig->fence;
     u64 target = sig->target;
     if (ID3D12Fence_GetCompletedValue(fence) < target) {
+        /* TODO: Reuse events from pool */
         HANDLE event = CreateEvent(0, 0, 0, 0);
         ID3D12Fence_SetEventOnCompletion(sig->fence, sig->target, event);
         WaitForSingleObject(event, INFINITE);
@@ -2213,6 +2225,11 @@ INTERNAL SYS_JOB_DEF(dx12_wait_fence_job, job)
     }
 }
 
+
+/* ========================== *
+ * Texture
+ * ========================== */
+
 struct gp_resource *gp_texture_alloc(enum gp_texture_format format, u32 flags, struct v2i32 size, void *initial_data, struct snc_counter *counter)
 {
     __prof;
@@ -2894,25 +2911,18 @@ void gp_present(struct gp_swapchain *gp_swapchain, struct v2i32 backbuffer_resol
  * Evictor thread
  * ========================== */
 
-INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
+INTERNAL SYS_JOB_DEF(dx12_evictor_job, _)
 {
     __prof;
-    (UNUSED)arg;
+    (UNUSED)_;
-    struct arena_temp scratch = scratch_begin_no_conflict();
-
-    HANDLE event = CreateEvent(0, 0, 0, 0);
-    HANDLE events[2] = ZI;
-    events[0] = G.evictor_thread_wake_event;
-    events[1] = event;
 
     u64 completed_targets[DX12_NUM_QUEUES] = ZI;
 
-    b32 shutdown = atomic32_fetch(&G.evictor_thread_shutdown);
+    b32 shutdown = 0;
     while (!shutdown) {
-        struct arena_temp temp = arena_temp_begin(scratch.arena);
         {
-            __profn("Run");
+            __profn("Dx12 evictor run");
-
+            struct arena_temp scratch = scratch_begin_no_conflict();
             u64 targets[countof(completed_targets)] = ZI;
 
             /* Copy queued data */
@@ -2922,7 +2932,7 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
                 __profn("Copy queued releases");
                 struct snc_lock lock = snc_lock_e(&G.fenced_releases_mutex);
                 num_fenced_releases = G.fenced_releases_arena->pos / sizeof(struct fenced_release_data);
-                fenced_releases = arena_push_array_no_zero(temp.arena, struct fenced_release_data, num_fenced_releases);
+                fenced_releases = arena_push_array_no_zero(scratch.arena, struct fenced_release_data, num_fenced_releases);
                 MEMCPY(fenced_releases, arena_base(G.fenced_releases_arena), G.fenced_releases_arena->pos);
                 arena_reset(G.fenced_releases_arena);
                 MEMCPY(targets, G.fenced_release_targets, sizeof(targets));
@@ -2932,16 +2942,21 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
             /* Wait until fences reach target */
             {
                 __profn("Check fences");
-                for (u32 i = 0; i < countof(targets) && !shutdown; ++i) {
+                for (u32 i = 0; i < countof(targets); ++i) {
-                    while (completed_targets[i] < targets[i] && !shutdown) {
+                    while (completed_targets[i] < targets[i]) {
                         struct command_queue *cq = G.command_queues[i];
                         completed_targets[i] = ID3D12Fence_GetCompletedValue(cq->submit_fence);
                         if (completed_targets[i] < targets[i]) {
-                            ID3D12Fence_SetEventOnCompletion(cq->submit_fence, targets[i], event);
-                            {
                             __profn("Wait on fence");
-                                WaitForMultipleObjects(2, events, 0, INFINITE);
+                            {
-                                shutdown = atomic32_fetch(&G.evictor_thread_shutdown);
+                                struct dx12_wait_fence_job_sig sig = ZI;
+                                sig.fence = cq->submit_fence;
+                                sig.target = targets[i];
+                                {
+                                    struct snc_counter counter = ZI;
+                                    sys_run(1, dx12_wait_fence_job, &sig, SYS_POOL_FLOATING, SYS_PRIORITY_LOW, &counter);
+                                    snc_counter_wait(&counter);
+                                }
                             }
                         }
                     }
@@ -2950,7 +2965,6 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
 
             /* Process releases */
             for (u32 i = 0; i < num_fenced_releases; ++i) {
-                __profn("Release");
                 struct fenced_release_data *fr = &fenced_releases[i];
                 switch (fr->kind) {
                     default:
@@ -2972,17 +2986,16 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
                     } break;
                 }
             }
-        }
-        arena_temp_end(temp);
-        {
-            __profn("Sleep");
-            WaitForSingleObject(G.evictor_thread_wake_event, INFINITE);
-            shutdown = atomic32_fetch(&G.evictor_thread_shutdown);
-        }
-    }
-
-    /* Release event */
-    CloseHandle(event);
-
             scratch_end(scratch);
         }
+        struct snc_lock lock = snc_lock_e(&G.evictor_wake_mutex);
+        {
+            while (!G.evictor_shutdown && G.evictor_wake_gen == 0) {
+                snc_cv_wait(&G.evictor_wake_cv, &lock);
+            }
+            shutdown = G.evictor_shutdown;
+            G.evictor_wake_gen = 0;
+        }
+        snc_unlock(&lock);
+    }
+}

src/sys.h

@@ -10,16 +10,22 @@
 
 
 /* ========================== *
- * On exit
+ * Exit
  * ========================== */
 
- /* Functions to be called for graceful shutdown (in reverse order) */
 
 #define SYS_EXIT_FUNC(name) void name(void)
 typedef SYS_EXIT_FUNC(sys_exit_func);
 
+/* Registers a function to be called during graceful shutdown (in reverse order) */
 void sys_on_exit(sys_exit_func *func);
 
+/* Signals the program to shut down gracefully and run exit callbacks */
+void sys_exit(void);
+
+/* Forcefully exits the program and displays `msg` to the user */
+void sys_panic(struct string msg);
+
 /* ========================== *
  * Scheduler
  * ========================== */
@@ -109,7 +115,7 @@ struct sys_scratch_ctx *sys_scratch_ctx_from_fiber_id(i16 fiber_id);
 
  /* Must be defined by app */
 
-void sys_app_entry(struct string args_str);
+void sys_app_startup(struct string args_str);
 
 
 
@@ -534,13 +540,5 @@ struct string sys_get_clipboard_text(struct arena *arena);
 
 void sys_true_rand(struct string b);
 u32 sys_num_logical_processors(void);
-void sys_exit(void);
-void sys_panic(struct string msg);
-
-/* ========================== *
- * Command line
- * ========================== */
-
-b32 sys_run_command(struct string cmd);
 
 #endif

src/sys_win32.c

@@ -263,10 +263,13 @@ GLOBAL struct {
 
     wchar_t cmdline_args_wstr[8192];
 
-    /* Panic */
+    /* Application control flow */
     struct atomic32 panicking;
     wchar_t panic_wstr[4096];
     HANDLE panic_event;
+    HANDLE startup_end_event;
+    HANDLE exit_begin_event;
+    HANDLE exit_end_event;
 
     /* Key lookup table */
     enum sys_btn vk_btn_table[256];
@@ -349,7 +352,7 @@ INTERNAL void tm_unlock(struct ticket_mutex *tm)
 
 
 /* ========================== *
- * On exit
+ * Exit
  * ========================== */
 
 void sys_on_exit(sys_exit_func *func)
@@ -361,9 +364,57 @@ void sys_on_exit(sys_exit_func *func)
     G.exit_funcs[index] = func;
 }
 
+void sys_exit(void)
+{
+    SetEvent(G.exit_begin_event);
+}
 
+void sys_panic(struct string msg)
+{
+    if (atomic32_fetch_test_set(&G.panicking, 0, 1) == 0) {
+        log_panic(msg);
 
+        wchar_t *wstr = G.panic_wstr;
+        u64 wstr_len = 0;
 
+        wchar_t prefix[] = L"A fatal error has occured and the application needs to exit:\n\n";
+        MEMCPY(wstr, prefix, min_u64(countof(G.panic_wstr), (countof(prefix) << 1)));
+        wstr_len += countof(prefix) - 1;
+
+        /* Perform manual string encode to avoid any implicit memory
+         * allocation (in case allocation is unreliable) */
+        struct string str8 = msg;
+        u64 pos8 = 0;
+        while (pos8 < str8.len) {
+            struct string str8_remaining = { .len = (str8.len - pos8), .text = str8.text + pos8 };
+            struct uni_decode_utf8_result decoded = uni_decode_utf8(str8_remaining);
+            struct uni_encode_utf16_result encoded = uni_encode_utf16(decoded.codepoint);
+            u64 wstr_new_len = wstr_len + encoded.count16;
+            if (wstr_new_len < (countof(G.panic_wstr) - 1)) {
+                u16 *dest = wstr + wstr_len;
+                MEMCPY(dest, encoded.chars16, (encoded.count16 << 1));
+                wstr_len = wstr_new_len;
+                pos8 += decoded.advance8;
+            } else {
+                break;
+            }
+        }
+
+        wstr[wstr_len] = 0;
+
+#if RTC
+        MessageBoxExW(0, wstr, L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
+        ASSERT(0);
+#endif
+
+        SetEvent(G.panic_event);
+
+        /* Wait for process termination */
+        if (GetCurrentThreadId() != G.main_thread_id) {
+            Sleep(INFINITE);
+        }
+    }
+}
 
 /* ========================== *
  * Scheduler
@@ -3093,91 +3144,34 @@ u32 sys_num_logical_processors(void)
     return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
 }
 
-void sys_exit(void)
-{
-    ExitProcess(1);
-}
-
-void sys_panic(struct string msg)
-{
-    if (atomic32_fetch_test_set(&G.panicking, 0, 1) == 0) {
-        log_panic(msg);
-
-        wchar_t *wstr = G.panic_wstr;
-        u64 wstr_len = 0;
-
-        wchar_t prefix[] = L"A fatal error has occured and the application needs to exit:\n\n";
-        MEMCPY(wstr, prefix, min_u64(countof(G.panic_wstr), (countof(prefix) << 1)));
-        wstr_len += countof(prefix) - 1;
-
-        /* Perform manual string encode to avoid any implicit memory
-         * allocation (in case allocation is unreliable) */
-        struct string str8 = msg;
-        u64 pos8 = 0;
-        while (pos8 < str8.len) {
-            struct string str8_remaining = { .len = (str8.len - pos8), .text = str8.text + pos8 };
-            struct uni_decode_utf8_result decoded = uni_decode_utf8(str8_remaining);
-            struct uni_encode_utf16_result encoded = uni_encode_utf16(decoded.codepoint);
-            u64 wstr_new_len = wstr_len + encoded.count16;
-            if (wstr_new_len < (countof(G.panic_wstr) - 1)) {
-                u16 *dest = wstr + wstr_len;
-                MEMCPY(dest, encoded.chars16, (encoded.count16 << 1));
-                wstr_len = wstr_new_len;
-                pos8 += decoded.advance8;
-            } else {
-                break;
-            }
-        }
-
-        wstr[wstr_len] = 0;
-
-#if RTC
-        MessageBoxExW(0, wstr, L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
-        ASSERT(0);
-#endif
-
-        WRITE_BARRIER();
-        SetEvent(G.panic_event);
-
-        /* Wait for process termination */
-        if (GetCurrentThreadId() != G.main_thread_id) {
-            Sleep(INFINITE);
-        }
-    }
-}
-
-/* ========================== *
- * Command line
- * ========================== */
-
-b32 sys_run_command(struct string cmd)
-{
-    b32 success = 0;
-    {
-        struct arena_temp scratch = scratch_begin_no_conflict();
-        wchar_t *cmd_wstr = wstr_from_string(scratch.arena, cmd);
-        STARTUPINFO si = ZI;
-        si.cb = sizeof(si);
-        PROCESS_INFORMATION pi = ZI;
-        success = CreateProcessW(0, cmd_wstr, 0, 0, 0, DETACHED_PROCESS, 0, 0, &si, &pi);
-        scratch_end(scratch);
-    }
-    return success;
-}
-
 /* ========================== *
  * Entry point
  * ========================== */
 
-INTERNAL SYS_THREAD_DEF(win32_app_thread_entry_point, arg)
+INTERNAL SYS_JOB_DEF(sys_app_startup_job, _)
 {
-    (UNUSED)arg;
+    (UNUSED)_;
     struct arena_temp scratch = scratch_begin_no_conflict();
+    {
         struct string cmdline_args = string_from_wstr(scratch.arena, G.cmdline_args_wstr, countof(G.cmdline_args_wstr));
-    sys_app_entry(cmdline_args);
+        sys_app_startup(cmdline_args);
+        SetEvent(G.startup_end_event);
+    }
     scratch_end(scratch);
 }
 
+INTERNAL SYS_JOB_DEF(sys_app_shutdown_job, _)
+{
+    __prof;
+    (UNUSED)_;
+    i32 num_funcs = atomic32_fetch(&G.num_exit_funcs);
+    for (i32 i = num_funcs - 1; i >= 0; --i) {
+        sys_exit_func *func = G.exit_funcs[i];
+        func();
+    }
+    SetEvent(G.exit_end_event);
+}
+
 int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, _In_ LPWSTR cmdline_wstr, _In_ int show_code)
 {
     (UNUSED)instance;
@@ -3204,7 +3198,15 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
 
     __profthread("Main thread", PROF_THREAD_GROUP_MAIN);
 
-    const wchar_t *error_msg = 0;
+    /* ========================== *
+     * Sys startup
+     * ========================== */
+
+    /* Set up exit events */
+    G.panic_event = CreateEventW(0, 1, 0, 0);
+    G.startup_end_event = CreateEventW(0, 1, 0, 0);
+    G.exit_begin_event = CreateEventW(0, 1, 0, 0);
+    G.exit_end_event = CreateEventW(0, 1, 0, 0);
 
     /* Init timer */
     {
@@ -3246,9 +3248,6 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
     G.main_thread_id = GetCurrentThreadId();
     SetThreadDescription(GetCurrentThread(), L"Main thread");
 
-    /* Set up panic event */
-    G.panic_event = CreateEventW(0, 1, 0, 0);
-
     /* Query system info */
     GetSystemInfo(&G.info);
 
@@ -3282,79 +3281,85 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
         ExtractIconExW(path, 0, &wc->hIcon, &wc->hIconSm, 1);
 
         if (!RegisterClassExW(wc)) {
-            /* TODO: GetLastError */
+            sys_panic(LIT("Failed to register window class"));
-            error_msg = L"Failed to register window class";
-            goto abort;
         }
     }
 
     /* Register raw input */
-    {
+    if (!atomic32_fetch(&G.panicking)) {
         RAWINPUTDEVICE rid = (RAWINPUTDEVICE) {
             .usUsagePage = 0x01,        /* HID_USAGE_PAGE_GENERIC */
             .usUsage = 0x02,            /* HID_USAGE_GENERIC_MOUSE */
             //.dwFlags = RIDEV_NOLEGACY   /* Adds mouse and also ignores legacy mouse messages */
         };
 
-        if (!RegisterRawInputDevices(&rid, 1, sizeof(rid))) {
+        b32 success = RegisterRawInputDevices(&rid, 1, sizeof(rid));
-            /* TODO: GetLastError */
+        ASSERT(success);
-            error_msg = L"Failed to register raw input device";
+        (UNUSED)success;
-            goto abort;
-        }
     }
 
     /* Start test thread */
-    struct sys_thread *test_thread = sys_thread_alloc(test_entry, 0, LIT("Test thread"), PROF_THREAD_GROUP_APP);
+    struct sys_thread *test_thread = 0;
+    if (!atomic32_fetch(&G.panicking)) {
+        test_thread = sys_thread_alloc(test_entry, 0, LIT("Test thread"), PROF_THREAD_GROUP_APP);
+    }
 
     /* ========================== *
-     * App thread setup
+     * App startup
      * ========================== */
 
-    /* Call app thread and wait for return */
+    /* Run app start job */
-    {
+    if (!atomic32_fetch(&G.panicking)) {
-        /* Start app thread */
+        sys_run(1, sys_app_startup_job, 0, SYS_POOL_FLOATING, SYS_PRIORITY_HIGH, 0);
-        struct sys_thread *app_thread = sys_thread_alloc(&win32_app_thread_entry_point, 0, LIT("App thread"), PROF_THREAD_GROUP_APP);
-
-        /* Get app thread handle */
-        HANDLE app_thread_handle = 0;
-        {
-            struct snc_lock lock = snc_lock_s(&G.threads_mutex);
-            struct win32_thread *wt = (struct win32_thread *)app_thread;
-            app_thread_handle = wt->handle;
-            snc_unlock(&lock);
     }
 
-
+    /* Wait for startup end or panic */
-        /* Wait for either app thread exit or panic */
+    if (!atomic32_fetch(&G.panicking)) {
-        if (app_thread_handle) {
+        HANDLE handles[] = {
-            HANDLE wait_handles[] = {
+            G.startup_end_event,
-                app_thread_handle,
             G.panic_event
         };
-            DWORD res = WaitForMultipleObjects(countof(wait_handles), wait_handles, 0, INFINITE);
+        WaitForMultipleObjects(countof(handles), handles, 0, INFINITE);
-            if (res == WAIT_OBJECT_0) {
-                sys_thread_force_release(app_thread);
-            }
-            ASSERT(res != WAIT_FAILED);
-            (UNUSED)res;
-        }
     }
 
-    /* Run exit callbacks */
+    /* Wait for exit start or panic */
-    {
+    if (!atomic32_fetch(&G.panicking)) {
-        __profn("Run exit callbacks");
+        HANDLE handles[] = {
-        i32 num_funcs = atomic32_fetch(&G.num_exit_funcs);
+            G.exit_begin_event,
-        for (i32 i = num_funcs - 1; i >= 0; --i) {
+            G.panic_event
-            sys_exit_func *func = G.exit_funcs[i];
+        };
-            func();
+        WaitForMultipleObjects(countof(handles), handles, 0, INFINITE);
-        }
     }
 
-    /* Signal shutdown */
+    /* ========================== *
+     * App shutdown
+     * ========================== */
+
+    /* Run exit callbacks job */
+    if (!atomic32_fetch(&G.panicking)) {
+        sys_run(1, sys_app_shutdown_job, 0, SYS_POOL_FLOATING, SYS_PRIORITY_HIGH, 0);
+    }
+
+    /* ========================== *
+     * Sys shutdown
+     * ========================== */
+
+    /* Wait for exit end or panic */
+    if (!atomic32_fetch(&G.panicking)) {
+        HANDLE handles[] = {
+            G.exit_end_event,
+            G.panic_event
+        };
+        WaitForMultipleObjects(countof(handles), handles, 0, INFINITE);
+    }
+
+    /* Signal sys shutdown */
+    if (!atomic32_fetch(&G.panicking)) {
         atomic32_fetch_set(&G.shutdown, 1);
+    }
 
     /* Shutdown test thread */
-    {
+    if (!atomic32_fetch(&G.panicking)) {
         for (enum sys_pool pool_kind = 0; pool_kind < (i32)countof(G.job_pools); ++pool_kind) {
             struct job_pool *pool = &G.job_pools[pool_kind];
             struct snc_lock lock = snc_lock_e(&pool->workers_wake_mutex);
@@ -3364,8 +3369,8 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
             }
             snc_unlock(&lock);
         }
-    }
         sys_thread_wait_release(test_thread);
+    }
 
     /* Find any dangling threads that haven't exited gracefully by now */
     if (!atomic32_fetch(&G.panicking)) {
@@ -3387,28 +3392,14 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
         snc_unlock(&lock);
     }
 
-    /* Check if panicking */
+    /* Exit */
+    i32 exit_code = 0;
     if (atomic32_fetch(&G.panicking)) {
-        /* Wait for panic message to be ready */
         WaitForSingleObject(G.panic_event, INFINITE);
-        /* Set error and abort */
+        MessageBoxExW(0, G.panic_wstr,  L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
-        error_msg = G.panic_wstr;
+        exit_code = 1;
-        goto abort;
     }
-
+    return exit_code;
-    /* ========================== *
-     * Abort
-     * ========================== */
-
-    abort:
-
-    if (error_msg) {
-        MessageBoxExW(0, error_msg,  L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
-        ASSERT(0);
-        return 1;
-    }
-
-    return 0;
 }
 
 /* ========================== *

src/user.c

@@ -714,13 +714,13 @@ INTERNAL void user_update(struct sys_window *window)
         for (u64 ent_index = 0; ent_index < events.count; ++ent_index) {
             struct sys_event *event = &events.events[ent_index];
             if (event->kind == SYS_EVENT_KIND_QUIT) {
-                app_exit();
+                sys_exit();
             }
 
             if (event->kind == SYS_EVENT_KIND_BUTTON_UP) {
                 /* Escape quit */
                 if (event->button == SYS_BTN_ESC) {
-                    app_exit();
+                    sys_exit();
                 }
             }