app startup as job
This commit is contained in:
parent
9fc74f4838
commit
2f3c47697b
21
src/app.c
21
src/app.c
@ -26,7 +26,6 @@
|
|||||||
GLOBAL struct {
|
GLOBAL struct {
|
||||||
struct arena *arena;
|
struct arena *arena;
|
||||||
struct string write_path;
|
struct string write_path;
|
||||||
struct snc_counter exit_fence;
|
|
||||||
} G = ZI, DEBUG_ALIAS(G, G_app);
|
} G = ZI, DEBUG_ALIAS(G, G_app);
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
@ -172,10 +171,10 @@ INTERNAL struct app_arg_list parse_args(struct arena *arena, struct string args_
|
|||||||
* Entry point
|
* Entry point
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
void sys_app_entry(struct string args_str)
|
void sys_app_startup(struct string args_str)
|
||||||
{
|
{
|
||||||
|
__prof;
|
||||||
struct arena_temp scratch = scratch_begin_no_conflict();
|
struct arena_temp scratch = scratch_begin_no_conflict();
|
||||||
snc_counter_add(&G.exit_fence, 1);
|
|
||||||
|
|
||||||
struct app_arg_list args = parse_args(scratch.arena, args_str);
|
struct app_arg_list args = parse_args(scratch.arena, args_str);
|
||||||
struct string logfile_name = LIT("log.log");
|
struct string logfile_name = LIT("log.log");
|
||||||
@ -278,9 +277,11 @@ void sys_app_entry(struct string args_str)
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* Startup systems */
|
/* Global systems */
|
||||||
resource_startup();
|
resource_startup();
|
||||||
gp_startup();
|
gp_startup();
|
||||||
|
|
||||||
|
/* Subsystems */
|
||||||
struct sock_startup_receipt sock_sr = sock_startup();
|
struct sock_startup_receipt sock_sr = sock_startup();
|
||||||
struct host_startup_receipt host_sr = host_startup(&sock_sr);
|
struct host_startup_receipt host_sr = host_startup(&sock_sr);
|
||||||
struct asset_cache_startup_receipt asset_cache_sr = asset_cache_startup();
|
struct asset_cache_startup_receipt asset_cache_sr = asset_cache_startup();
|
||||||
@ -291,15 +292,14 @@ void sys_app_entry(struct string args_str)
|
|||||||
struct sound_startup_receipt sound_sr = sound_startup(&asset_cache_sr);
|
struct sound_startup_receipt sound_sr = sound_startup(&asset_cache_sr);
|
||||||
struct draw_startup_receipt draw_sr = draw_startup(&font_sr);
|
struct draw_startup_receipt draw_sr = draw_startup(&font_sr);
|
||||||
struct sim_startup_receipt sim_sr = sim_startup();
|
struct sim_startup_receipt sim_sr = sim_startup();
|
||||||
|
|
||||||
|
/* Interface systems */
|
||||||
struct playback_startup_receipt playback_sr = playback_startup(&mixer_sr);
|
struct playback_startup_receipt playback_sr = playback_startup(&mixer_sr);
|
||||||
struct user_startup_receipt user_sr = user_startup(&font_sr, &sprite_sr, &draw_sr, &asset_cache_sr, &sound_sr, &mixer_sr, &host_sr, &sim_sr, connect_address);
|
struct user_startup_receipt user_sr = user_startup(&font_sr, &sprite_sr, &draw_sr, &asset_cache_sr, &sound_sr, &mixer_sr, &host_sr, &sim_sr, connect_address);
|
||||||
|
|
||||||
(UNUSED)user_sr;
|
(UNUSED)user_sr;
|
||||||
(UNUSED)playback_sr;
|
(UNUSED)playback_sr;
|
||||||
|
|
||||||
/* Wait for app_exit() */
|
|
||||||
snc_counter_wait(&G.exit_fence);
|
|
||||||
|
|
||||||
#if 0
|
#if 0
|
||||||
/* Write window settings to file */
|
/* Write window settings to file */
|
||||||
{
|
{
|
||||||
@ -326,11 +326,6 @@ void sys_app_entry(struct string args_str)
|
|||||||
sys_window_release(window);
|
sys_window_release(window);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
logf_info("Program exited normally");
|
//logf_info("Program exited normally");
|
||||||
scratch_end(scratch);
|
scratch_end(scratch);
|
||||||
}
|
}
|
||||||
|
|
||||||
void app_exit(void)
|
|
||||||
{
|
|
||||||
snc_counter_add(&G.exit_fence, -1);
|
|
||||||
}
|
|
||||||
|
|||||||
@ -3,6 +3,4 @@
|
|||||||
|
|
||||||
struct string app_write_path_cat(struct arena *arena, struct string filename);
|
struct string app_write_path_cat(struct arena *arena, struct string filename);
|
||||||
|
|
||||||
void app_exit(void);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@ -42,7 +42,7 @@
|
|||||||
#define SIM_TILES_PER_UNIT_SQRT (4)
|
#define SIM_TILES_PER_UNIT_SQRT (4)
|
||||||
#define SIM_TILES_PER_CHUNK_SQRT (16)
|
#define SIM_TILES_PER_CHUNK_SQRT (16)
|
||||||
|
|
||||||
#define SIM_TICKS_PER_SECOND 100
|
#define SIM_TICKS_PER_SECOND 50
|
||||||
//#define SIM_TIMESCALE 1
|
//#define SIM_TIMESCALE 1
|
||||||
/* Like USER_INTERP_RATIO, but applies to snapshots received by the local sim from the
|
/* Like USER_INTERP_RATIO, but applies to snapshots received by the local sim from the
|
||||||
* master sim (how far back in time should the client render the server's state) */
|
* master sim (how far back in time should the client render the server's state) */
|
||||||
|
|||||||
@ -326,10 +326,12 @@ GLOBAL struct {
|
|||||||
struct snc_mutex global_submit_mutex;
|
struct snc_mutex global_submit_mutex;
|
||||||
struct command_queue *command_queues[DX12_NUM_QUEUES];
|
struct command_queue *command_queues[DX12_NUM_QUEUES];
|
||||||
|
|
||||||
/* Evictor thread */
|
/* Evictor job */
|
||||||
struct atomic32 evictor_thread_shutdown;
|
struct snc_counter evictor_job_counter;
|
||||||
HANDLE evictor_thread_wake_event;
|
struct snc_cv evictor_wake_cv;
|
||||||
struct sys_thread *evictor_thread;
|
struct snc_mutex evictor_wake_mutex;
|
||||||
|
i64 evictor_wake_gen;
|
||||||
|
b32 evictor_shutdown;
|
||||||
} G = ZI, DEBUG_ALIAS(G, G_gp_dx12);
|
} G = ZI, DEBUG_ALIAS(G, G_gp_dx12);
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
@ -343,7 +345,7 @@ INTERNAL void dx12_init_pipelines(void);
|
|||||||
INTERNAL struct cpu_descriptor_heap *cpu_descriptor_heap_alloc(enum D3D12_DESCRIPTOR_HEAP_TYPE type);
|
INTERNAL struct cpu_descriptor_heap *cpu_descriptor_heap_alloc(enum D3D12_DESCRIPTOR_HEAP_TYPE type);
|
||||||
INTERNAL struct command_queue *command_queue_alloc(enum D3D12_COMMAND_LIST_TYPE type, enum D3D12_COMMAND_QUEUE_PRIORITY priority, struct string dbg_name);
|
INTERNAL struct command_queue *command_queue_alloc(enum D3D12_COMMAND_LIST_TYPE type, enum D3D12_COMMAND_QUEUE_PRIORITY priority, struct string dbg_name);
|
||||||
INTERNAL void command_queue_release(struct command_queue *cq);
|
INTERNAL void command_queue_release(struct command_queue *cq);
|
||||||
INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg);
|
INTERNAL SYS_JOB_DEF(dx12_evictor_job, _);
|
||||||
INTERNAL void fenced_release(void *data, enum fenced_release_kind kind);
|
INTERNAL void fenced_release(void *data, enum fenced_release_kind kind);
|
||||||
|
|
||||||
#if RESOURCE_RELOADING
|
#if RESOURCE_RELOADING
|
||||||
@ -390,9 +392,8 @@ void gp_startup(void)
|
|||||||
#endif
|
#endif
|
||||||
sys_on_exit(gp_shutdown);
|
sys_on_exit(gp_shutdown);
|
||||||
|
|
||||||
/* Start evictor thread */
|
/* Start evictor job */
|
||||||
G.evictor_thread_wake_event = CreateEvent(0, 0, 0, 0);
|
sys_run(1, dx12_evictor_job, 0, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &G.evictor_job_counter);
|
||||||
G.evictor_thread = sys_thread_alloc(evictor_thread_entry_point, 0, LIT("GPU resource evictor thread"), PROF_THREAD_GROUP_EVICTORS);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
INTERNAL SYS_EXIT_FUNC(gp_shutdown)
|
INTERNAL SYS_EXIT_FUNC(gp_shutdown)
|
||||||
@ -410,9 +411,13 @@ INTERNAL SYS_EXIT_FUNC(gp_shutdown)
|
|||||||
(UNUSED)command_queue_release;
|
(UNUSED)command_queue_release;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
atomic32_fetch_set(&G.evictor_thread_shutdown, 1);
|
{
|
||||||
SetEvent(G.evictor_thread_wake_event);
|
struct snc_lock lock = snc_lock_e(&G.evictor_wake_mutex);
|
||||||
sys_thread_wait_release(G.evictor_thread);
|
G.evictor_shutdown = 1;
|
||||||
|
snc_cv_signal(&G.evictor_wake_cv, I32_MAX);
|
||||||
|
snc_unlock(&lock);
|
||||||
|
}
|
||||||
|
snc_counter_wait(&G.evictor_job_counter);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
@ -723,7 +728,6 @@ INTERNAL HRESULT dx12_include_open(ID3DInclude *d3d_handler, D3D_INCLUDE_TYPE in
|
|||||||
struct resource *res = &handler->open_resources[handler->num_open_resources++];
|
struct resource *res = &handler->open_resources[handler->num_open_resources++];
|
||||||
*res = resource_open(name);
|
*res = resource_open(name);
|
||||||
if (resource_exists(res)) {
|
if (resource_exists(res)) {
|
||||||
++handler->num_open_resources;
|
|
||||||
struct string data = resource_get_data(res);
|
struct string data = resource_get_data(res);
|
||||||
*data_out = data.text;
|
*data_out = data.text;
|
||||||
*data_len_out = data.len;
|
*data_len_out = data.len;
|
||||||
@ -748,6 +752,7 @@ INTERNAL HRESULT dx12_include_close(ID3DInclude *d3d_handler, LPCVOID data)
|
|||||||
|
|
||||||
INTERNAL struct dx12_include_handler *dx12_include_handler_alloc(struct arena *arena, struct pipeline *pipeline)
|
INTERNAL struct dx12_include_handler *dx12_include_handler_alloc(struct arena *arena, struct pipeline *pipeline)
|
||||||
{
|
{
|
||||||
|
__prof;
|
||||||
struct dx12_include_handler *handler = arena_push(arena, struct dx12_include_handler);
|
struct dx12_include_handler *handler = arena_push(arena, struct dx12_include_handler);
|
||||||
handler->d3d_handler.lpVtbl = &handler->vtbl;
|
handler->d3d_handler.lpVtbl = &handler->vtbl;
|
||||||
handler->vtbl.Open = dx12_include_open;
|
handler->vtbl.Open = dx12_include_open;
|
||||||
@ -758,6 +763,7 @@ INTERNAL struct dx12_include_handler *dx12_include_handler_alloc(struct arena *a
|
|||||||
|
|
||||||
INTERNAL void dx12_include_handler_release(struct dx12_include_handler *handler)
|
INTERNAL void dx12_include_handler_release(struct dx12_include_handler *handler)
|
||||||
{
|
{
|
||||||
|
__prof;
|
||||||
for (u64 i = 0; i < handler->num_open_resources; ++i) {
|
for (u64 i = 0; i < handler->num_open_resources; ++i) {
|
||||||
ASSERT(0); /* Resource should have been closed by handler by now */
|
ASSERT(0); /* Resource should have been closed by handler by now */
|
||||||
struct resource *res = &handler->open_resources[i];
|
struct resource *res = &handler->open_resources[i];
|
||||||
@ -1566,7 +1572,12 @@ INTERNAL void fenced_release(void *data, enum fenced_release_kind kind)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Wake evictor */
|
/* Wake evictor */
|
||||||
SetEvent(G.evictor_thread_wake_event);
|
struct snc_lock lock = snc_lock_e(&G.evictor_wake_mutex);
|
||||||
|
{
|
||||||
|
++G.evictor_wake_gen;
|
||||||
|
snc_cv_signal(&G.evictor_wake_cv, I32_MAX);
|
||||||
|
}
|
||||||
|
snc_unlock(&lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
@ -2191,7 +2202,7 @@ INTERNAL D3D12_INDEX_BUFFER_VIEW ibv_from_command_buffer(struct command_buffer *
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* Texture
|
* Wait job
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
struct dx12_wait_fence_job_sig {
|
struct dx12_wait_fence_job_sig {
|
||||||
@ -2206,6 +2217,7 @@ INTERNAL SYS_JOB_DEF(dx12_wait_fence_job, job)
|
|||||||
ID3D12Fence *fence = sig->fence;
|
ID3D12Fence *fence = sig->fence;
|
||||||
u64 target = sig->target;
|
u64 target = sig->target;
|
||||||
if (ID3D12Fence_GetCompletedValue(fence) < target) {
|
if (ID3D12Fence_GetCompletedValue(fence) < target) {
|
||||||
|
/* TODO: Reuse events from pool */
|
||||||
HANDLE event = CreateEvent(0, 0, 0, 0);
|
HANDLE event = CreateEvent(0, 0, 0, 0);
|
||||||
ID3D12Fence_SetEventOnCompletion(sig->fence, sig->target, event);
|
ID3D12Fence_SetEventOnCompletion(sig->fence, sig->target, event);
|
||||||
WaitForSingleObject(event, INFINITE);
|
WaitForSingleObject(event, INFINITE);
|
||||||
@ -2213,6 +2225,11 @@ INTERNAL SYS_JOB_DEF(dx12_wait_fence_job, job)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* ========================== *
|
||||||
|
* Texture
|
||||||
|
* ========================== */
|
||||||
|
|
||||||
struct gp_resource *gp_texture_alloc(enum gp_texture_format format, u32 flags, struct v2i32 size, void *initial_data, struct snc_counter *counter)
|
struct gp_resource *gp_texture_alloc(enum gp_texture_format format, u32 flags, struct v2i32 size, void *initial_data, struct snc_counter *counter)
|
||||||
{
|
{
|
||||||
__prof;
|
__prof;
|
||||||
@ -2894,25 +2911,18 @@ void gp_present(struct gp_swapchain *gp_swapchain, struct v2i32 backbuffer_resol
|
|||||||
* Evictor thread
|
* Evictor thread
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
|
INTERNAL SYS_JOB_DEF(dx12_evictor_job, _)
|
||||||
{
|
{
|
||||||
__prof;
|
__prof;
|
||||||
(UNUSED)arg;
|
(UNUSED)_;
|
||||||
struct arena_temp scratch = scratch_begin_no_conflict();
|
|
||||||
|
|
||||||
HANDLE event = CreateEvent(0, 0, 0, 0);
|
|
||||||
HANDLE events[2] = ZI;
|
|
||||||
events[0] = G.evictor_thread_wake_event;
|
|
||||||
events[1] = event;
|
|
||||||
|
|
||||||
u64 completed_targets[DX12_NUM_QUEUES] = ZI;
|
u64 completed_targets[DX12_NUM_QUEUES] = ZI;
|
||||||
|
|
||||||
b32 shutdown = atomic32_fetch(&G.evictor_thread_shutdown);
|
b32 shutdown = 0;
|
||||||
while (!shutdown) {
|
while (!shutdown) {
|
||||||
struct arena_temp temp = arena_temp_begin(scratch.arena);
|
|
||||||
{
|
{
|
||||||
__profn("Run");
|
__profn("Dx12 evictor run");
|
||||||
|
struct arena_temp scratch = scratch_begin_no_conflict();
|
||||||
u64 targets[countof(completed_targets)] = ZI;
|
u64 targets[countof(completed_targets)] = ZI;
|
||||||
|
|
||||||
/* Copy queued data */
|
/* Copy queued data */
|
||||||
@ -2922,7 +2932,7 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
|
|||||||
__profn("Copy queued releases");
|
__profn("Copy queued releases");
|
||||||
struct snc_lock lock = snc_lock_e(&G.fenced_releases_mutex);
|
struct snc_lock lock = snc_lock_e(&G.fenced_releases_mutex);
|
||||||
num_fenced_releases = G.fenced_releases_arena->pos / sizeof(struct fenced_release_data);
|
num_fenced_releases = G.fenced_releases_arena->pos / sizeof(struct fenced_release_data);
|
||||||
fenced_releases = arena_push_array_no_zero(temp.arena, struct fenced_release_data, num_fenced_releases);
|
fenced_releases = arena_push_array_no_zero(scratch.arena, struct fenced_release_data, num_fenced_releases);
|
||||||
MEMCPY(fenced_releases, arena_base(G.fenced_releases_arena), G.fenced_releases_arena->pos);
|
MEMCPY(fenced_releases, arena_base(G.fenced_releases_arena), G.fenced_releases_arena->pos);
|
||||||
arena_reset(G.fenced_releases_arena);
|
arena_reset(G.fenced_releases_arena);
|
||||||
MEMCPY(targets, G.fenced_release_targets, sizeof(targets));
|
MEMCPY(targets, G.fenced_release_targets, sizeof(targets));
|
||||||
@ -2932,16 +2942,21 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
|
|||||||
/* Wait until fences reach target */
|
/* Wait until fences reach target */
|
||||||
{
|
{
|
||||||
__profn("Check fences");
|
__profn("Check fences");
|
||||||
for (u32 i = 0; i < countof(targets) && !shutdown; ++i) {
|
for (u32 i = 0; i < countof(targets); ++i) {
|
||||||
while (completed_targets[i] < targets[i] && !shutdown) {
|
while (completed_targets[i] < targets[i]) {
|
||||||
struct command_queue *cq = G.command_queues[i];
|
struct command_queue *cq = G.command_queues[i];
|
||||||
completed_targets[i] = ID3D12Fence_GetCompletedValue(cq->submit_fence);
|
completed_targets[i] = ID3D12Fence_GetCompletedValue(cq->submit_fence);
|
||||||
if (completed_targets[i] < targets[i]) {
|
if (completed_targets[i] < targets[i]) {
|
||||||
ID3D12Fence_SetEventOnCompletion(cq->submit_fence, targets[i], event);
|
__profn("Wait on fence");
|
||||||
{
|
{
|
||||||
__profn("Wait on fence");
|
struct dx12_wait_fence_job_sig sig = ZI;
|
||||||
WaitForMultipleObjects(2, events, 0, INFINITE);
|
sig.fence = cq->submit_fence;
|
||||||
shutdown = atomic32_fetch(&G.evictor_thread_shutdown);
|
sig.target = targets[i];
|
||||||
|
{
|
||||||
|
struct snc_counter counter = ZI;
|
||||||
|
sys_run(1, dx12_wait_fence_job, &sig, SYS_POOL_FLOATING, SYS_PRIORITY_LOW, &counter);
|
||||||
|
snc_counter_wait(&counter);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2950,7 +2965,6 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
|
|||||||
|
|
||||||
/* Process releases */
|
/* Process releases */
|
||||||
for (u32 i = 0; i < num_fenced_releases; ++i) {
|
for (u32 i = 0; i < num_fenced_releases; ++i) {
|
||||||
__profn("Release");
|
|
||||||
struct fenced_release_data *fr = &fenced_releases[i];
|
struct fenced_release_data *fr = &fenced_releases[i];
|
||||||
switch (fr->kind) {
|
switch (fr->kind) {
|
||||||
default:
|
default:
|
||||||
@ -2972,17 +2986,16 @@ INTERNAL SYS_THREAD_DEF(evictor_thread_entry_point, arg)
|
|||||||
} break;
|
} break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
scratch_end(scratch);
|
||||||
}
|
}
|
||||||
arena_temp_end(temp);
|
struct snc_lock lock = snc_lock_e(&G.evictor_wake_mutex);
|
||||||
{
|
{
|
||||||
__profn("Sleep");
|
while (!G.evictor_shutdown && G.evictor_wake_gen == 0) {
|
||||||
WaitForSingleObject(G.evictor_thread_wake_event, INFINITE);
|
snc_cv_wait(&G.evictor_wake_cv, &lock);
|
||||||
shutdown = atomic32_fetch(&G.evictor_thread_shutdown);
|
}
|
||||||
|
shutdown = G.evictor_shutdown;
|
||||||
|
G.evictor_wake_gen = 0;
|
||||||
}
|
}
|
||||||
|
snc_unlock(&lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Release event */
|
|
||||||
CloseHandle(event);
|
|
||||||
|
|
||||||
scratch_end(scratch);
|
|
||||||
}
|
}
|
||||||
|
|||||||
20
src/sys.h
20
src/sys.h
@ -10,16 +10,22 @@
|
|||||||
|
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* On exit
|
* Exit
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
/* Functions to be called for graceful shutdown (in reverse order) */
|
|
||||||
|
|
||||||
#define SYS_EXIT_FUNC(name) void name(void)
|
#define SYS_EXIT_FUNC(name) void name(void)
|
||||||
typedef SYS_EXIT_FUNC(sys_exit_func);
|
typedef SYS_EXIT_FUNC(sys_exit_func);
|
||||||
|
|
||||||
|
/* Registers a function to be called during graceful shutdown (in reverse order) */
|
||||||
void sys_on_exit(sys_exit_func *func);
|
void sys_on_exit(sys_exit_func *func);
|
||||||
|
|
||||||
|
/* Signals the program to shut down gracefully and run exit callbacks */
|
||||||
|
void sys_exit(void);
|
||||||
|
|
||||||
|
/* Forcefully exits the program and displays `msg` to the user */
|
||||||
|
void sys_panic(struct string msg);
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* Scheduler
|
* Scheduler
|
||||||
* ========================== */
|
* ========================== */
|
||||||
@ -109,7 +115,7 @@ struct sys_scratch_ctx *sys_scratch_ctx_from_fiber_id(i16 fiber_id);
|
|||||||
|
|
||||||
/* Must be defined by app */
|
/* Must be defined by app */
|
||||||
|
|
||||||
void sys_app_entry(struct string args_str);
|
void sys_app_startup(struct string args_str);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -534,13 +540,5 @@ struct string sys_get_clipboard_text(struct arena *arena);
|
|||||||
|
|
||||||
void sys_true_rand(struct string b);
|
void sys_true_rand(struct string b);
|
||||||
u32 sys_num_logical_processors(void);
|
u32 sys_num_logical_processors(void);
|
||||||
void sys_exit(void);
|
|
||||||
void sys_panic(struct string msg);
|
|
||||||
|
|
||||||
/* ========================== *
|
|
||||||
* Command line
|
|
||||||
* ========================== */
|
|
||||||
|
|
||||||
b32 sys_run_command(struct string cmd);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
295
src/sys_win32.c
295
src/sys_win32.c
@ -263,10 +263,13 @@ GLOBAL struct {
|
|||||||
|
|
||||||
wchar_t cmdline_args_wstr[8192];
|
wchar_t cmdline_args_wstr[8192];
|
||||||
|
|
||||||
/* Panic */
|
/* Application control flow */
|
||||||
struct atomic32 panicking;
|
struct atomic32 panicking;
|
||||||
wchar_t panic_wstr[4096];
|
wchar_t panic_wstr[4096];
|
||||||
HANDLE panic_event;
|
HANDLE panic_event;
|
||||||
|
HANDLE startup_end_event;
|
||||||
|
HANDLE exit_begin_event;
|
||||||
|
HANDLE exit_end_event;
|
||||||
|
|
||||||
/* Key lookup table */
|
/* Key lookup table */
|
||||||
enum sys_btn vk_btn_table[256];
|
enum sys_btn vk_btn_table[256];
|
||||||
@ -349,7 +352,7 @@ INTERNAL void tm_unlock(struct ticket_mutex *tm)
|
|||||||
|
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* On exit
|
* Exit
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
void sys_on_exit(sys_exit_func *func)
|
void sys_on_exit(sys_exit_func *func)
|
||||||
@ -361,9 +364,57 @@ void sys_on_exit(sys_exit_func *func)
|
|||||||
G.exit_funcs[index] = func;
|
G.exit_funcs[index] = func;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void sys_exit(void)
|
||||||
|
{
|
||||||
|
SetEvent(G.exit_begin_event);
|
||||||
|
}
|
||||||
|
|
||||||
|
void sys_panic(struct string msg)
|
||||||
|
{
|
||||||
|
if (atomic32_fetch_test_set(&G.panicking, 0, 1) == 0) {
|
||||||
|
log_panic(msg);
|
||||||
|
|
||||||
|
wchar_t *wstr = G.panic_wstr;
|
||||||
|
u64 wstr_len = 0;
|
||||||
|
|
||||||
|
wchar_t prefix[] = L"A fatal error has occured and the application needs to exit:\n\n";
|
||||||
|
MEMCPY(wstr, prefix, min_u64(countof(G.panic_wstr), (countof(prefix) << 1)));
|
||||||
|
wstr_len += countof(prefix) - 1;
|
||||||
|
|
||||||
|
/* Perform manual string encode to avoid any implicit memory
|
||||||
|
* allocation (in case allocation is unreliable) */
|
||||||
|
struct string str8 = msg;
|
||||||
|
u64 pos8 = 0;
|
||||||
|
while (pos8 < str8.len) {
|
||||||
|
struct string str8_remaining = { .len = (str8.len - pos8), .text = str8.text + pos8 };
|
||||||
|
struct uni_decode_utf8_result decoded = uni_decode_utf8(str8_remaining);
|
||||||
|
struct uni_encode_utf16_result encoded = uni_encode_utf16(decoded.codepoint);
|
||||||
|
u64 wstr_new_len = wstr_len + encoded.count16;
|
||||||
|
if (wstr_new_len < (countof(G.panic_wstr) - 1)) {
|
||||||
|
u16 *dest = wstr + wstr_len;
|
||||||
|
MEMCPY(dest, encoded.chars16, (encoded.count16 << 1));
|
||||||
|
wstr_len = wstr_new_len;
|
||||||
|
pos8 += decoded.advance8;
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wstr[wstr_len] = 0;
|
||||||
|
|
||||||
|
#if RTC
|
||||||
|
MessageBoxExW(0, wstr, L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
|
||||||
|
ASSERT(0);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
SetEvent(G.panic_event);
|
||||||
|
|
||||||
|
/* Wait for process termination */
|
||||||
|
if (GetCurrentThreadId() != G.main_thread_id) {
|
||||||
|
Sleep(INFINITE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* Scheduler
|
* Scheduler
|
||||||
@ -3093,91 +3144,34 @@ u32 sys_num_logical_processors(void)
|
|||||||
return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
|
return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
|
||||||
}
|
}
|
||||||
|
|
||||||
void sys_exit(void)
|
|
||||||
{
|
|
||||||
ExitProcess(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
void sys_panic(struct string msg)
|
|
||||||
{
|
|
||||||
if (atomic32_fetch_test_set(&G.panicking, 0, 1) == 0) {
|
|
||||||
log_panic(msg);
|
|
||||||
|
|
||||||
wchar_t *wstr = G.panic_wstr;
|
|
||||||
u64 wstr_len = 0;
|
|
||||||
|
|
||||||
wchar_t prefix[] = L"A fatal error has occured and the application needs to exit:\n\n";
|
|
||||||
MEMCPY(wstr, prefix, min_u64(countof(G.panic_wstr), (countof(prefix) << 1)));
|
|
||||||
wstr_len += countof(prefix) - 1;
|
|
||||||
|
|
||||||
/* Perform manual string encode to avoid any implicit memory
|
|
||||||
* allocation (in case allocation is unreliable) */
|
|
||||||
struct string str8 = msg;
|
|
||||||
u64 pos8 = 0;
|
|
||||||
while (pos8 < str8.len) {
|
|
||||||
struct string str8_remaining = { .len = (str8.len - pos8), .text = str8.text + pos8 };
|
|
||||||
struct uni_decode_utf8_result decoded = uni_decode_utf8(str8_remaining);
|
|
||||||
struct uni_encode_utf16_result encoded = uni_encode_utf16(decoded.codepoint);
|
|
||||||
u64 wstr_new_len = wstr_len + encoded.count16;
|
|
||||||
if (wstr_new_len < (countof(G.panic_wstr) - 1)) {
|
|
||||||
u16 *dest = wstr + wstr_len;
|
|
||||||
MEMCPY(dest, encoded.chars16, (encoded.count16 << 1));
|
|
||||||
wstr_len = wstr_new_len;
|
|
||||||
pos8 += decoded.advance8;
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
wstr[wstr_len] = 0;
|
|
||||||
|
|
||||||
#if RTC
|
|
||||||
MessageBoxExW(0, wstr, L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
|
|
||||||
ASSERT(0);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
WRITE_BARRIER();
|
|
||||||
SetEvent(G.panic_event);
|
|
||||||
|
|
||||||
/* Wait for process termination */
|
|
||||||
if (GetCurrentThreadId() != G.main_thread_id) {
|
|
||||||
Sleep(INFINITE);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ========================== *
|
|
||||||
* Command line
|
|
||||||
* ========================== */
|
|
||||||
|
|
||||||
b32 sys_run_command(struct string cmd)
|
|
||||||
{
|
|
||||||
b32 success = 0;
|
|
||||||
{
|
|
||||||
struct arena_temp scratch = scratch_begin_no_conflict();
|
|
||||||
wchar_t *cmd_wstr = wstr_from_string(scratch.arena, cmd);
|
|
||||||
STARTUPINFO si = ZI;
|
|
||||||
si.cb = sizeof(si);
|
|
||||||
PROCESS_INFORMATION pi = ZI;
|
|
||||||
success = CreateProcessW(0, cmd_wstr, 0, 0, 0, DETACHED_PROCESS, 0, 0, &si, &pi);
|
|
||||||
scratch_end(scratch);
|
|
||||||
}
|
|
||||||
return success;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* Entry point
|
* Entry point
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
INTERNAL SYS_THREAD_DEF(win32_app_thread_entry_point, arg)
|
INTERNAL SYS_JOB_DEF(sys_app_startup_job, _)
|
||||||
{
|
{
|
||||||
(UNUSED)arg;
|
(UNUSED)_;
|
||||||
struct arena_temp scratch = scratch_begin_no_conflict();
|
struct arena_temp scratch = scratch_begin_no_conflict();
|
||||||
struct string cmdline_args = string_from_wstr(scratch.arena, G.cmdline_args_wstr, countof(G.cmdline_args_wstr));
|
{
|
||||||
sys_app_entry(cmdline_args);
|
struct string cmdline_args = string_from_wstr(scratch.arena, G.cmdline_args_wstr, countof(G.cmdline_args_wstr));
|
||||||
|
sys_app_startup(cmdline_args);
|
||||||
|
SetEvent(G.startup_end_event);
|
||||||
|
}
|
||||||
scratch_end(scratch);
|
scratch_end(scratch);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
INTERNAL SYS_JOB_DEF(sys_app_shutdown_job, _)
|
||||||
|
{
|
||||||
|
__prof;
|
||||||
|
(UNUSED)_;
|
||||||
|
i32 num_funcs = atomic32_fetch(&G.num_exit_funcs);
|
||||||
|
for (i32 i = num_funcs - 1; i >= 0; --i) {
|
||||||
|
sys_exit_func *func = G.exit_funcs[i];
|
||||||
|
func();
|
||||||
|
}
|
||||||
|
SetEvent(G.exit_end_event);
|
||||||
|
}
|
||||||
|
|
||||||
int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, _In_ LPWSTR cmdline_wstr, _In_ int show_code)
|
int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance, _In_ LPWSTR cmdline_wstr, _In_ int show_code)
|
||||||
{
|
{
|
||||||
(UNUSED)instance;
|
(UNUSED)instance;
|
||||||
@ -3204,7 +3198,15 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
|
|||||||
|
|
||||||
__profthread("Main thread", PROF_THREAD_GROUP_MAIN);
|
__profthread("Main thread", PROF_THREAD_GROUP_MAIN);
|
||||||
|
|
||||||
const wchar_t *error_msg = 0;
|
/* ========================== *
|
||||||
|
* Sys startup
|
||||||
|
* ========================== */
|
||||||
|
|
||||||
|
/* Set up exit events */
|
||||||
|
G.panic_event = CreateEventW(0, 1, 0, 0);
|
||||||
|
G.startup_end_event = CreateEventW(0, 1, 0, 0);
|
||||||
|
G.exit_begin_event = CreateEventW(0, 1, 0, 0);
|
||||||
|
G.exit_end_event = CreateEventW(0, 1, 0, 0);
|
||||||
|
|
||||||
/* Init timer */
|
/* Init timer */
|
||||||
{
|
{
|
||||||
@ -3246,9 +3248,6 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
|
|||||||
G.main_thread_id = GetCurrentThreadId();
|
G.main_thread_id = GetCurrentThreadId();
|
||||||
SetThreadDescription(GetCurrentThread(), L"Main thread");
|
SetThreadDescription(GetCurrentThread(), L"Main thread");
|
||||||
|
|
||||||
/* Set up panic event */
|
|
||||||
G.panic_event = CreateEventW(0, 1, 0, 0);
|
|
||||||
|
|
||||||
/* Query system info */
|
/* Query system info */
|
||||||
GetSystemInfo(&G.info);
|
GetSystemInfo(&G.info);
|
||||||
|
|
||||||
@ -3282,79 +3281,85 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
|
|||||||
ExtractIconExW(path, 0, &wc->hIcon, &wc->hIconSm, 1);
|
ExtractIconExW(path, 0, &wc->hIcon, &wc->hIconSm, 1);
|
||||||
|
|
||||||
if (!RegisterClassExW(wc)) {
|
if (!RegisterClassExW(wc)) {
|
||||||
/* TODO: GetLastError */
|
sys_panic(LIT("Failed to register window class"));
|
||||||
error_msg = L"Failed to register window class";
|
|
||||||
goto abort;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Register raw input */
|
/* Register raw input */
|
||||||
{
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
RAWINPUTDEVICE rid = (RAWINPUTDEVICE) {
|
RAWINPUTDEVICE rid = (RAWINPUTDEVICE) {
|
||||||
.usUsagePage = 0x01, /* HID_USAGE_PAGE_GENERIC */
|
.usUsagePage = 0x01, /* HID_USAGE_PAGE_GENERIC */
|
||||||
.usUsage = 0x02, /* HID_USAGE_GENERIC_MOUSE */
|
.usUsage = 0x02, /* HID_USAGE_GENERIC_MOUSE */
|
||||||
//.dwFlags = RIDEV_NOLEGACY /* Adds mouse and also ignores legacy mouse messages */
|
//.dwFlags = RIDEV_NOLEGACY /* Adds mouse and also ignores legacy mouse messages */
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!RegisterRawInputDevices(&rid, 1, sizeof(rid))) {
|
b32 success = RegisterRawInputDevices(&rid, 1, sizeof(rid));
|
||||||
/* TODO: GetLastError */
|
ASSERT(success);
|
||||||
error_msg = L"Failed to register raw input device";
|
(UNUSED)success;
|
||||||
goto abort;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Start test thread */
|
/* Start test thread */
|
||||||
struct sys_thread *test_thread = sys_thread_alloc(test_entry, 0, LIT("Test thread"), PROF_THREAD_GROUP_APP);
|
struct sys_thread *test_thread = 0;
|
||||||
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
|
test_thread = sys_thread_alloc(test_entry, 0, LIT("Test thread"), PROF_THREAD_GROUP_APP);
|
||||||
|
}
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
* App thread setup
|
* App startup
|
||||||
* ========================== */
|
* ========================== */
|
||||||
|
|
||||||
/* Call app thread and wait for return */
|
/* Run app start job */
|
||||||
{
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
/* Start app thread */
|
sys_run(1, sys_app_startup_job, 0, SYS_POOL_FLOATING, SYS_PRIORITY_HIGH, 0);
|
||||||
struct sys_thread *app_thread = sys_thread_alloc(&win32_app_thread_entry_point, 0, LIT("App thread"), PROF_THREAD_GROUP_APP);
|
|
||||||
|
|
||||||
/* Get app thread handle */
|
|
||||||
HANDLE app_thread_handle = 0;
|
|
||||||
{
|
|
||||||
struct snc_lock lock = snc_lock_s(&G.threads_mutex);
|
|
||||||
struct win32_thread *wt = (struct win32_thread *)app_thread;
|
|
||||||
app_thread_handle = wt->handle;
|
|
||||||
snc_unlock(&lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/* Wait for either app thread exit or panic */
|
|
||||||
if (app_thread_handle) {
|
|
||||||
HANDLE wait_handles[] = {
|
|
||||||
app_thread_handle,
|
|
||||||
G.panic_event
|
|
||||||
};
|
|
||||||
DWORD res = WaitForMultipleObjects(countof(wait_handles), wait_handles, 0, INFINITE);
|
|
||||||
if (res == WAIT_OBJECT_0) {
|
|
||||||
sys_thread_force_release(app_thread);
|
|
||||||
}
|
|
||||||
ASSERT(res != WAIT_FAILED);
|
|
||||||
(UNUSED)res;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Run exit callbacks */
|
/* Wait for startup end or panic */
|
||||||
{
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
__profn("Run exit callbacks");
|
HANDLE handles[] = {
|
||||||
i32 num_funcs = atomic32_fetch(&G.num_exit_funcs);
|
G.startup_end_event,
|
||||||
for (i32 i = num_funcs - 1; i >= 0; --i) {
|
G.panic_event
|
||||||
sys_exit_func *func = G.exit_funcs[i];
|
};
|
||||||
func();
|
WaitForMultipleObjects(countof(handles), handles, 0, INFINITE);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Signal shutdown */
|
/* Wait for exit start or panic */
|
||||||
atomic32_fetch_set(&G.shutdown, 1);
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
|
HANDLE handles[] = {
|
||||||
|
G.exit_begin_event,
|
||||||
|
G.panic_event
|
||||||
|
};
|
||||||
|
WaitForMultipleObjects(countof(handles), handles, 0, INFINITE);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ========================== *
|
||||||
|
* App shutdown
|
||||||
|
* ========================== */
|
||||||
|
|
||||||
|
/* Run exit callbacks job */
|
||||||
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
|
sys_run(1, sys_app_shutdown_job, 0, SYS_POOL_FLOATING, SYS_PRIORITY_HIGH, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ========================== *
|
||||||
|
* Sys shutdown
|
||||||
|
* ========================== */
|
||||||
|
|
||||||
|
/* Wait for exit end or panic */
|
||||||
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
|
HANDLE handles[] = {
|
||||||
|
G.exit_end_event,
|
||||||
|
G.panic_event
|
||||||
|
};
|
||||||
|
WaitForMultipleObjects(countof(handles), handles, 0, INFINITE);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Signal sys shutdown */
|
||||||
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
|
atomic32_fetch_set(&G.shutdown, 1);
|
||||||
|
}
|
||||||
|
|
||||||
/* Shutdown test thread */
|
/* Shutdown test thread */
|
||||||
{
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
for (enum sys_pool pool_kind = 0; pool_kind < (i32)countof(G.job_pools); ++pool_kind) {
|
for (enum sys_pool pool_kind = 0; pool_kind < (i32)countof(G.job_pools); ++pool_kind) {
|
||||||
struct job_pool *pool = &G.job_pools[pool_kind];
|
struct job_pool *pool = &G.job_pools[pool_kind];
|
||||||
struct snc_lock lock = snc_lock_e(&pool->workers_wake_mutex);
|
struct snc_lock lock = snc_lock_e(&pool->workers_wake_mutex);
|
||||||
@ -3364,8 +3369,8 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
|
|||||||
}
|
}
|
||||||
snc_unlock(&lock);
|
snc_unlock(&lock);
|
||||||
}
|
}
|
||||||
|
sys_thread_wait_release(test_thread);
|
||||||
}
|
}
|
||||||
sys_thread_wait_release(test_thread);
|
|
||||||
|
|
||||||
/* Find any dangling threads that haven't exited gracefully by now */
|
/* Find any dangling threads that haven't exited gracefully by now */
|
||||||
if (!atomic32_fetch(&G.panicking)) {
|
if (!atomic32_fetch(&G.panicking)) {
|
||||||
@ -3387,28 +3392,14 @@ int CALLBACK wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev_instance,
|
|||||||
snc_unlock(&lock);
|
snc_unlock(&lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Check if panicking */
|
/* Exit */
|
||||||
|
i32 exit_code = 0;
|
||||||
if (atomic32_fetch(&G.panicking)) {
|
if (atomic32_fetch(&G.panicking)) {
|
||||||
/* Wait for panic message to be ready */
|
|
||||||
WaitForSingleObject(G.panic_event, INFINITE);
|
WaitForSingleObject(G.panic_event, INFINITE);
|
||||||
/* Set error and abort */
|
MessageBoxExW(0, G.panic_wstr, L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
|
||||||
error_msg = G.panic_wstr;
|
exit_code = 1;
|
||||||
goto abort;
|
|
||||||
}
|
}
|
||||||
|
return exit_code;
|
||||||
/* ========================== *
|
|
||||||
* Abort
|
|
||||||
* ========================== */
|
|
||||||
|
|
||||||
abort:
|
|
||||||
|
|
||||||
if (error_msg) {
|
|
||||||
MessageBoxExW(0, error_msg, L"Fatal error", MB_ICONSTOP | MB_SETFOREGROUND | MB_TOPMOST, 0);
|
|
||||||
ASSERT(0);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ========================== *
|
/* ========================== *
|
||||||
|
|||||||
@ -714,13 +714,13 @@ INTERNAL void user_update(struct sys_window *window)
|
|||||||
for (u64 ent_index = 0; ent_index < events.count; ++ent_index) {
|
for (u64 ent_index = 0; ent_index < events.count; ++ent_index) {
|
||||||
struct sys_event *event = &events.events[ent_index];
|
struct sys_event *event = &events.events[ent_index];
|
||||||
if (event->kind == SYS_EVENT_KIND_QUIT) {
|
if (event->kind == SYS_EVENT_KIND_QUIT) {
|
||||||
app_exit();
|
sys_exit();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (event->kind == SYS_EVENT_KIND_BUTTON_UP) {
|
if (event->kind == SYS_EVENT_KIND_BUTTON_UP) {
|
||||||
/* Escape quit */
|
/* Escape quit */
|
||||||
if (event->button == SYS_BTN_ESC) {
|
if (event->button == SYS_BTN_ESC) {
|
||||||
app_exit();
|
sys_exit();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user