From e13aca535ae2f7014afcfb0dcb678a811af5886a Mon Sep 17 00:00:00 2001 From: jacob Date: Fri, 11 Jul 2025 12:34:05 -0500 Subject: [PATCH] 8 & 16 bit atomics --- src/atomic.h | 12 ++++++++++++ src/common.h | 26 ++++++++++++++++++++++++-- src/sys_win32.c | 24 ++++++++++++++---------- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/atomic.h b/src/atomic.h index e4ccd369..50c32f90 100644 --- a/src/atomic.h +++ b/src/atomic.h @@ -3,6 +3,18 @@ #if PLATFORM_WINDOWS +FORCE_INLINE i8 atomic8_fetch(struct atomic8 *x) { return (i8)_InterlockedCompareExchange8((char *)&x->_v, 0, 0); } +FORCE_INLINE i8 atomic8_fetch_set(struct atomic8 *x, i8 e) { return (i8)_InterlockedExchange8((char *)&x->_v, e); } +FORCE_INLINE i8 atomic8_fetch_test_set(struct atomic8 *x, i8 c, i8 e) { return (i8)_InterlockedCompareExchange8((char *)&x->_v, e, c); } +FORCE_INLINE i8 atomic8_fetch_xor(struct atomic8 *x, i8 c) { return (i8)_InterlockedXor8((char *)&x->_v, c); } +FORCE_INLINE i8 atomic8_fetch_add(struct atomic8 *x, i8 a) { return (i8)_InterlockedExchangeAdd8((char *)&x->_v, a); } + +FORCE_INLINE i16 atomic16_fetch(struct atomic16 *x) { return (i16)_InterlockedCompareExchange16(&x->_v, 0, 0); } +FORCE_INLINE i16 atomic16_fetch_set(struct atomic16 *x, i16 e) { return (i16)_InterlockedExchange16(&x->_v, e); } +FORCE_INLINE i16 atomic16_fetch_test_set(struct atomic16 *x, i16 c, i16 e) { return (i16)_InterlockedCompareExchange16(&x->_v, e, c); } +FORCE_INLINE i16 atomic16_fetch_xor(struct atomic16 *x, i16 c) { return (i16)_InterlockedXor16(&x->_v, c); } +FORCE_INLINE i16 atomic16_fetch_add(struct atomic16 *x, i16 a) { return (i16)_InterlockedExchangeAdd16(&x->_v, a); } + FORCE_INLINE i32 atomic32_fetch(struct atomic32 *x) { return (i32)_InterlockedCompareExchange((volatile long *)&x->_v, 0, 0); } FORCE_INLINE i32 atomic32_fetch_set(struct atomic32 *x, i32 e) { return (i32)_InterlockedExchange((volatile long *)&x->_v, e); } FORCE_INLINE i32 atomic32_fetch_test_set(struct atomic32 *x, i32 c, i32 e) { return (i32)_InterlockedCompareExchange((volatile long *)&x->_v, e, c); } diff --git a/src/common.h b/src/common.h index ac7e62f9..c1c802c4 100644 --- a/src/common.h +++ b/src/common.h @@ -277,9 +277,9 @@ void __asan_unpoison_memory_region(void const volatile *add, size_t); #define FIELD_SIZEOF(type, field) sizeof(((type *)0)->field) #if COMPILER_MSVC && !defined _CRT_USE_BUILTIN_OFFSETOF -# define FIELD_OFFSETOF(type, field) ((u64)&(((type *)0)->field)) +# define offsetof(type, field) ((u64)&(((type *)0)->field)) #else -# define FIELD_OFFSETOF(type, field) __builtin_offsetof(type, field) +# define offsetof(type, field) __builtin_offsetof(type, field) #endif /* Array */ @@ -391,6 +391,16 @@ GLOBAL const f64 *_f64_nan = (f64 *)&_f64_nan_u64; * Atomics * ========================== */ +/* NOTE: Must be aligned to 32 bit boundary by user */ +struct atomic8 { + volatile i8 _v; +}; + +/* NOTE: Must be aligned to 32 bit boundary by user */ +struct atomic16 { + volatile i16 _v; +}; + struct atomic32 { volatile i32 _v; }; @@ -403,6 +413,18 @@ struct atomic64 { * Cache-line isolated atomics * ========================== */ +struct alignas(64) atomic8_padded { + struct atomic8 v; + u8 _pad[60]; +}; +STATIC_ASSERT(sizeof(struct atomic8_padded) == 64 && alignof(struct atomic8_padded) == 64); + +struct alignas(64) atomic16_padded { + struct atomic16 v; + u8 _pad[60]; +}; +STATIC_ASSERT(sizeof(struct atomic16_padded) == 64 && alignof(struct atomic16_padded) == 64); + struct alignas(64) atomic32_padded { struct atomic32 v; u8 _pad[60]; diff --git a/src/sys_win32.c b/src/sys_win32.c index 7b2ce81a..84f8372e 100644 --- a/src/sys_win32.c +++ b/src/sys_win32.c @@ -162,9 +162,10 @@ struct alignas(64) fiber { /* ==================================================== */ char *name_cstr; /* 08 bytes */ /* ==================================================== */ + struct atomic16 wake_lock; /* 02 bytes (aligned) */ i16 id; /* 02 bytes */ i16 parent_id; /* 02 bytes */ - struct atomic32 wake_lock; /* 04 bytes */ + i16 can_yield; /* 02 bytes */ /* ==================================================== */ u64 wait_addr; /* 08 bytes */ /* ==================================================== */ @@ -204,6 +205,7 @@ struct alignas(64) fiber { }; STATIC_ASSERT(sizeof(struct fiber) == 128); /* Padding validation (increase if necessary) */ STATIC_ASSERT(alignof(struct fiber) == 64); /* Avoid false sharing */ +STATIC_ASSERT(offsetof(struct fiber, wake_lock) % 4 == 0); /* Atomic must be aligned */ STATIC_ASSERT(SYS_MAX_FIBERS < I16_MAX); /* Max fibers should fit in fiber id */ struct alignas(64) worker_ctx { @@ -392,8 +394,7 @@ i64 sys_current_scheduler_period_ns(void) void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns) { struct fiber *fiber = fiber_from_id(sys_current_fiber_id()); - i16 parent_fiber_id = fiber->parent_id; - if (parent_fiber_id > 0) { + if (fiber->can_yield) { *fiber->yield_param = (struct yield_param) { .kind = YIELD_KIND_WAIT, .wait = { @@ -403,8 +404,8 @@ void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns) .timeout_ns = timeout_ns } }; - struct fiber *parent_fiber = fiber_from_id(parent_fiber_id); - job_fiber_yield(fiber, parent_fiber); + ASSERT(fiber->parent_id != 0); + job_fiber_yield(fiber, fiber_from_id(fiber->parent_id)); } else { i32 timeout_ms = 0; if (timeout_ns == I64_MAX) { @@ -564,7 +565,7 @@ INTERNAL void wake_fibers_locked(i32 num_fibers, struct fiber **fibers) queue->first = info; } queue->last = info; - atomic32_fetch_set(&fiber->wake_lock, 0); + atomic16_fetch_set(&fiber->wake_lock, 0); } tm_unlock(&queue->lock); } @@ -611,7 +612,7 @@ INTERNAL void wake_address(void *addr, i32 count) if (wait_addr_list) { fibers = arena_push_array_no_zero(scratch.arena, struct fiber *, wait_addr_list->num_waiters); for (struct fiber *fiber = fiber_from_id(wait_addr_list->first_waiter); fiber && num_fibers < count; fiber = fiber_from_id(fiber->next_addr_waiter)) { - if (atomic32_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) { + if (atomic16_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) { fibers[num_fibers] = fiber; ++num_fibers; } @@ -660,7 +661,7 @@ INTERNAL void wake_time(u64 time) /* Set waiter wake status & build fibers list */ fibers = arena_push_array_no_zero(scratch.arena, struct fiber *, wait_time_list->num_waiters); for (struct fiber *fiber = fiber_from_id(wait_time_list->first_waiter); fiber; fiber = fiber_from_id(fiber->next_time_waiter)) { - if (atomic32_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) { + if (atomic16_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) { fibers[num_fibers] = fiber; ++num_fibers; } @@ -753,12 +754,15 @@ INTERNAL struct fiber *fiber_alloc(enum fiber_kind kind) /* Init win32 fiber */ if (kind == FIBER_KIND_JOB_WORKER) { fiber->addr = CreateFiber(FIBER_STACK_SIZE, job_fiber_entry, (void *)(i64)fiber_id); + fiber->can_yield = 1; } else { fiber->addr = ConvertThreadToFiber((void *)(i64)fiber_id); + fiber->can_yield = 0; } } - MEMZERO_STRUCT(&fiber->wait_addr); - MEMZERO_STRUCT(&fiber->wait_time); + MEMZERO_STRUCT(&fiber->wake_lock); + fiber->wait_addr = 0; + fiber->wait_time = 0; fiber->prev_addr_waiter = 0; fiber->next_addr_waiter = 0; fiber->prev_time_waiter = 0;