diff --git a/src/resource.c b/src/resource.c index 3f973acf..b567d605 100644 --- a/src/resource.c +++ b/src/resource.c @@ -3,6 +3,7 @@ #include "arena.h" #include "tar.h" #include "incbin.h" +#include "util.h" /* ========================== * * Global data @@ -22,9 +23,16 @@ GLOBAL struct { #endif #if RESOURCE_RELOADING - struct sys_thread resource_watch_thread; + struct sys_thread resource_watch_monitor_thread; + struct sys_thread resource_watch_dispatch_thread; + + struct sys_mutex watch_dispatcher_mutex; + struct arena watch_dispatcher_info_arena; + struct sys_watch_info_list watch_dispatcher_info_list; + struct sys_condition_variable watch_dispatcher_cv; + b32 watch_dispatcher_shutdown; + struct sys_mutex watch_callbacks_mutex; - b32 watch_stop; resource_watch_callback *watch_callbacks[64]; u64 num_watch_callbacks; #endif @@ -35,7 +43,8 @@ GLOBAL struct { * ========================== */ #if RESOURCE_RELOADING -INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(resource_watch_thread_entry_point, _); +INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(resource_watch_monitor_thread_entry_point, _); +INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(resource_watch_dispatcher_thread_entry_point, _); INTERNAL APP_EXIT_CALLBACK_FUNC_DEF(resource_shutdown); #endif @@ -58,8 +67,14 @@ struct resource_startup_receipt resource_startup(void) #if RESOURCE_RELOADING G.watch_callbacks_mutex = sys_mutex_alloc(); + + G.watch_dispatcher_mutex = sys_mutex_alloc(); + G.watch_dispatcher_info_arena = arena_alloc(GIGABYTE(64)); + G.watch_dispatcher_cv = sys_condition_variable_alloc(); + app_register_exit_callback(&resource_shutdown); - G.resource_watch_thread = sys_thread_alloc(resource_watch_thread_entry_point, NULL, LIT("[P2] Resource watcher")); + G.resource_watch_monitor_thread = sys_thread_alloc(resource_watch_monitor_thread_entry_point, NULL, LIT("[P2] Resource watch monitor")); + G.resource_watch_dispatch_thread = sys_thread_alloc(resource_watch_dispatcher_thread_entry_point, NULL, LIT("[P2] Resource watch dispatcher")); #endif @@ -132,12 +147,14 @@ b32 resource_exists(struct string path) INTERNAL APP_EXIT_CALLBACK_FUNC_DEF(resource_shutdown) { __prof; - /* Wait until any watch callbacks finish before shutting down */ - struct sys_lock lock = sys_mutex_lock_e(&G.watch_callbacks_mutex); + /* Wait for dispatcher thread to finish before shutting down */ + struct sys_lock lock = sys_mutex_lock_e(&G.watch_dispatcher_mutex); { - G.watch_stop = true; + G.watch_dispatcher_shutdown = true; } sys_mutex_unlock(&lock); + sys_condition_variable_broadcast(&G.watch_dispatcher_cv); + sys_thread_wait_release(&G.resource_watch_dispatch_thread); } void resource_register_watch_callback(resource_watch_callback *callback) @@ -153,34 +170,94 @@ void resource_register_watch_callback(resource_watch_callback *callback) sys_mutex_unlock(&lock); } -INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(resource_watch_thread_entry_point, _) +INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(resource_watch_monitor_thread_entry_point, _) { (UNUSED)_; struct temp_arena scratch = scratch_begin_no_conflict(); - struct sys_watch dir_watch = sys_watch_alloc(LIT("res")); + struct sys_watch watch = sys_watch_alloc(LIT("res")); - /* NOTE: This thread is force-shutdown at the moment, however shutdown is guaranteed not to occur while the watch mutex is locked by the thread */ + /* NOTE: We let OS force-shutdown this thread */ volatile i32 run = true; while (run) { struct temp_arena temp = arena_temp_begin(scratch.arena); - struct sys_watch_info *info = sys_watch_wait(temp.arena, &dir_watch); + struct sys_watch_info_list res = sys_watch_wait(temp.arena, &watch); + struct sys_lock lock = sys_mutex_lock_e(&G.watch_dispatcher_mutex); { - struct sys_lock lock = sys_mutex_lock_s(&G.watch_callbacks_mutex); - if (!G.watch_stop) { - while (info) { - for (u64 i = 0; i < G.num_watch_callbacks; ++i) { - resource_watch_callback *callback = G.watch_callbacks[i]; - callback(info); - } - info = info->next; - } - } - sys_mutex_unlock(&lock); + G.watch_dispatcher_info_list = sys_watch_info_copy(&G.watch_dispatcher_info_arena, res); } + sys_mutex_unlock(&lock); + sys_condition_variable_broadcast(&G.watch_dispatcher_cv); arena_temp_end(temp); } - sys_watch_release(&dir_watch); + sys_watch_release(&watch); + scratch_end(scratch); +} + +/* NOTE: We separate the responsibilities of monitoring directory changes + * & dispatching watch callbacks into two separate threads so that we can delay + * the dispatch of these callbacks, allowing for deduplication of file + * modification notifications. */ + +#define WATCH_DISPATCHER_DELAY_SECONDS 0.100 +#define WATCH_DISPATCHER_DEDUP_DICT_BINS 128 + +INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(resource_watch_dispatcher_thread_entry_point, _) +{ + (UNUSED)_; + struct temp_arena scratch = scratch_begin_no_conflict(); + + struct sys_lock watch_dispatcher_lock = sys_mutex_lock_e(&G.watch_dispatcher_mutex); + while (!G.watch_dispatcher_shutdown) { + if (G.watch_dispatcher_info_arena.pos > 0) { + /* Unlock and sleep a bit so duplicate events pile up */ + { + sys_mutex_unlock(&watch_dispatcher_lock); + sys_sleep(WATCH_DISPATCHER_DELAY_SECONDS); + watch_dispatcher_lock = sys_mutex_lock_e(&G.watch_dispatcher_mutex); + } + if (!G.watch_dispatcher_shutdown) { + struct temp_arena temp = arena_temp_begin(scratch.arena); + + /* Pull watch info from queue */ + struct sys_watch_info_list watch_info_list = sys_watch_info_copy(temp.arena, G.watch_dispatcher_info_list); + arena_reset(&G.watch_dispatcher_info_arena); + + /* Unlock and run callbacks */ + sys_mutex_unlock(&watch_dispatcher_lock); + { + struct fixed_dict dedup_dict = fixed_dict_init(temp.arena, WATCH_DISPATCHER_DEDUP_DICT_BINS); + for (struct sys_watch_info *info = watch_info_list.first; info; info = info->next) { + b32 skip = false; + if (info->kind == SYS_WATCH_INFO_KIND_MODIFIED) { + /* Skip modified notifications for the same file */ + if ((u64)fixed_dict_get(&dedup_dict, info->name) != 1) { + fixed_dict_set(temp.arena, &dedup_dict, info->name, (void *)1); + } else { + skip = true; + } + } + if (!skip) { + struct sys_lock callbacks_lock = sys_mutex_lock_s(&G.watch_callbacks_mutex); + for (u64 i = 0; i < G.num_watch_callbacks; ++i) { + resource_watch_callback *callback = G.watch_callbacks[i]; + callback(info); + } + sys_mutex_unlock(&callbacks_lock); + } + } + } + watch_dispatcher_lock = sys_mutex_lock_e(&G.watch_dispatcher_mutex); + + arena_temp_end(temp); + } + } + + if (!G.watch_dispatcher_shutdown) { + sys_condition_variable_wait(&G.watch_dispatcher_cv, &watch_dispatcher_lock); + } + } + scratch_end(scratch); } diff --git a/src/sprite.c b/src/sprite.c index 829ab5f0..a25697ca 100644 --- a/src/sprite.c +++ b/src/sprite.c @@ -20,10 +20,10 @@ #define MAX_LOADER_THREADS 4 /* How long between evictor thread scans */ -#define EVICTOR_CYCLE_INTERVAL (RESOURCE_RELOADING ? 0.100 : 0.500) +#define EVICTOR_CYCLE_INTERVAL_NS NS_FROM_SECONDS(0.5) /* Time a cache entry spends unused until it's considered evictable (rounded up to multiple of of EVICTOR_CYCLE_INTERVAL) */ -#define EVICTOR_GRACE_PERIOD 10.000 +#define EVICTOR_GRACE_PERIOD_NS NS_FROM_SECONDS(10) #define TCTX_ARENA_RESERVE MEGABYTE(64) @@ -94,8 +94,6 @@ struct cache_node { #if RESOURCE_RELOADING struct atomic_i32 out_of_date; /* Has the resource changed since this node was loaded */ - u64 tag_path_len; - u8 tag_path[4096]; #endif }; @@ -317,7 +315,7 @@ b32 sprite_tag_eq(struct sprite_tag t1, struct sprite_tag t2) INTERNAL struct cache_node_hash cache_node_hash_from_tag_hash(u64 tag_hash, enum cache_node_kind kind) { - return (struct cache_node_hash) { .v = hash_fnv64(tag_hash, STRING(1, (u8 *)&kind)) }; + return (struct cache_node_hash) { .v = rand_u64_from_seed(tag_hash + kind) }; } /* ========================== * @@ -355,7 +353,7 @@ INTERNAL void cache_node_load_texture(struct cache_node *n, struct sprite_tag ta atomic_u32_eval_exchange(&n->state, CACHE_NODE_STATE_WORKING); struct string path = tag.path; - logf_info("Loading sprite texture \"%F\"", FMT_STR(path)); + logf_info("Loading sprite texture [%F] \"%F\"", FMT_HEX(n->hash.v), FMT_STR(path)); i64 start_ns = sys_time_ns(); ASSERT(string_ends_with(path, LIT(".ase"))); @@ -383,20 +381,16 @@ INTERNAL void cache_node_load_texture(struct cache_node *n, struct sprite_tag ta /* TODO: Query renderer for more accurate texture size in VRAM */ memory_size += (decoded.image.width * decoded.image.height) * sizeof(*decoded.image.pixels); } else { - logf_error("Sprite \"%F\" not found", FMT_STR(path)); + logf_error("Sprite [%F] \"%F\" not found", FMT_HEX(n->hash.v), FMT_STR(path)); } -#if RESOURCE_RELOADING - u64 cpy_len = min_u64(tag.path.len, ARRAY_COUNT(n->tag_path)); - n->tag_path_len = cpy_len; - MEMCPY(n->tag_path, tag.path.text, cpy_len); -#endif } arena_set_readonly(&n->arena); n->memory_usage = n->arena.committed + memory_size; atomic_u64_eval_add_u64(&G.cache.memory_usage, n->memory_usage); f64 elapsed = SECONDS_FROM_NS(sys_time_ns() - start_ns); - logf_info("Finished loading sprite texture \"%F\" in %F seconds (cache size: %F bytes).", + logf_info("Finished loading sprite texture [%F] \"%F\" in %F seconds (cache size: %F bytes).", + FMT_HEX(n->hash.v), FMT_STR(path), FMT_FLOAT(elapsed), FMT_UINT(n->memory_usage)); @@ -658,7 +652,7 @@ INTERNAL void cache_node_load_sheet(struct cache_node *n, struct sprite_tag tag) atomic_u32_eval_exchange(&n->state, CACHE_NODE_STATE_WORKING); struct string path = tag.path; - logf_info("Loading sprite sheet \"%F\"", FMT_STR(path)); + logf_info("Loading sprite sheet [%F] \"%F\"", FMT_HEX(n->hash.v), FMT_STR(path)); i64 start_ns = sys_time_ns(); //ASSERT(string_ends_with(path, LIT(".ase"))); @@ -683,17 +677,13 @@ INTERNAL void cache_node_load_sheet(struct cache_node *n, struct sprite_tag tag) logf_error("Sprite \"%F\" not found", FMT_STR(path)); } } -#if RESOURCE_RELOADING - u64 cpy_len = min_u64(tag.path.len, ARRAY_COUNT(n->tag_path)); - n->tag_path_len = cpy_len; - MEMCPY(n->tag_path, tag.path.text, cpy_len); -#endif arena_set_readonly(&n->arena); n->memory_usage = n->arena.committed; atomic_u64_eval_add_u64(&G.cache.memory_usage, n->memory_usage); f64 elapsed = SECONDS_FROM_NS(sys_time_ns() - start_ns); - logf_info("Finished loading sprite sheet \"%F\" in %F seconds (cache size: %F bytes).", + logf_info("Finished loading sprite sheet [%F] \"%F\" in %F seconds (cache size: %F bytes).", + FMT_HEX(n->hash.v), FMT_STR(path), FMT_FLOAT(elapsed), FMT_UINT(n->memory_usage)); @@ -1090,7 +1080,7 @@ INTERNAL RESOURCE_WATCH_CALLBACK_FUNC_DEF(sprite_resource_watch_callback, info) { struct string name = info->name; struct sprite_tag tag = sprite_tag_from_path(name); - for (u64 kind = 0; kind < NUM_CACHE_NODE_KINDS; ++kind) { + for (enum cache_node_kind kind = 0; kind < NUM_CACHE_NODE_KINDS; ++kind) { struct cache_node_hash hash = cache_node_hash_from_tag_hash(tag.hash, kind); u64 cache_bin_index = hash.v % CACHE_BINS_COUNT; struct cache_bin *bin = &G.cache.bins[cache_bin_index]; @@ -1154,14 +1144,13 @@ INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(sprite_evictor_thread_entry_point, arg) #if RESOURCE_RELOADING /* Check if file changed for resource reloading */ if (!consider_for_eviction) { - struct string path = STRING(n->tag_path_len, n->tag_path); if (atomic_i32_eval(&n->out_of_date)) { switch (n->kind) { case CACHE_NODE_KIND_TEXTURE: { - logf_info("Resource file for sprite texture \"%F\" has changed. Evicting to allow for reloading.", FMT_STR(path)); + logf_info("Resource file for sprite texture [%F] has changed. Evicting to allow for reloading.", FMT_HEX(n->hash.v)); } break; case CACHE_NODE_KIND_SHEET: { - logf_info("Resource file for sprite sheet \"%F\" has changed. Evicting to allow for reloading.", FMT_STR(path)); + logf_info("Resource file for sprite sheet [%F] has changed. Evicting to allow for reloading.", FMT_HEX(n->hash.v)); } break; default: { sys_panic(LIT("Unknown sprite cache node kind")); } break; } @@ -1173,8 +1162,8 @@ INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(sprite_evictor_thread_entry_point, arg) /* Check usage time */ u32 last_used_cycle = refcount.last_modified_cycle; - f64 time_since_use = (f64)(cur_cycle - last_used_cycle) * EVICTOR_CYCLE_INTERVAL; - if (time_since_use > EVICTOR_GRACE_PERIOD) { + i64 time_since_use_ns = ((i64)cur_cycle - (i64)last_used_cycle) * EVICTOR_CYCLE_INTERVAL_NS; + if (time_since_use_ns > EVICTOR_GRACE_PERIOD_NS) { /* Cache is over budget and node hasn't been referenced in a while */ consider_for_eviction = true; } @@ -1285,7 +1274,7 @@ INTERNAL SYS_THREAD_ENTRY_POINT_FUNC_DEF(sprite_evictor_thread_entry_point, arg) scratch_end(scratch); /* Wait */ - sys_condition_variable_wait_time(&G.evictor_cv, &evictor_lock, EVICTOR_CYCLE_INTERVAL); + sys_condition_variable_wait_time(&G.evictor_cv, &evictor_lock, SECONDS_FROM_NS(EVICTOR_CYCLE_INTERVAL_NS)); } sys_mutex_unlock(&evictor_lock); } diff --git a/src/sys.h b/src/sys.h index 958b9017..4c7398dc 100644 --- a/src/sys.h +++ b/src/sys.h @@ -276,11 +276,18 @@ struct sys_watch_info { enum sys_watch_info_kind kind; struct string name; struct sys_watch_info *next; + struct sys_watch_info *prev; +}; + +struct sys_watch_info_list { + struct sys_watch_info *first; + struct sys_watch_info *last; }; struct sys_watch sys_watch_alloc(struct string path); void sys_watch_release(struct sys_watch *dw); -struct sys_watch_info *sys_watch_wait(struct arena *arena, struct sys_watch *dw); +struct sys_watch_info_list sys_watch_wait(struct arena *arena, struct sys_watch *dw); +struct sys_watch_info_list sys_watch_info_copy(struct arena *arena, struct sys_watch_info_list src); /* ========================== * * Window diff --git a/src/sys_win32.c b/src/sys_win32.c index 039864a9..0f5501a0 100644 --- a/src/sys_win32.c +++ b/src/sys_win32.c @@ -748,12 +748,11 @@ void sys_watch_release(struct sys_watch *dw) sys_mutex_unlock(&lock); } -struct sys_watch_info *sys_watch_wait(struct arena *arena, struct sys_watch *dw) +struct sys_watch_info_list sys_watch_wait(struct arena *arena, struct sys_watch *dw) { __prof; struct win32_watch *w32_watch = (struct win32_watch *)dw->handle; - struct sys_watch_info *first_info = NULL; - struct sys_watch_info *last_info = NULL; + struct sys_watch_info_list list = ZI; DWORD filter = FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME | @@ -781,10 +780,12 @@ struct sys_watch_info *sys_watch_wait(struct arena *arena, struct sys_watch *dw) FILE_NOTIFY_INFORMATION *res = (FILE_NOTIFY_INFORMATION *)(w32_watch->results_buff + offset); struct sys_watch_info *info = arena_push_zero(arena, struct sys_watch_info); - if (last_info) { - last_info->next = info; + if (list.last) { + list.last->next = info; + info->prev = list.last; + list.last = info; } else { - first_info = info; + list.first = info; } struct string16 name16 = ZI; @@ -840,7 +841,25 @@ struct sys_watch_info *sys_watch_wait(struct arena *arena, struct sys_watch *dw) } } - return first_info; + return list; +} + +struct sys_watch_info_list sys_watch_info_copy(struct arena *arena, struct sys_watch_info_list src_list) +{ + struct sys_watch_info_list dst_list = ZI; + for (struct sys_watch_info *src = src_list.first; src; src = src->next) { + struct sys_watch_info *dst = arena_push_zero(arena, struct sys_watch_info); + dst->kind = src->kind; + dst->name = string_copy(arena, src->name); + if (dst_list.last) { + dst_list.last->next = dst; + dst->prev = dst_list.last; + dst_list.last = dst; + } else { + dst_list.first = dst; + } + } + return dst_list; } /* ========================== *