8 & 16 bit atomics

src/atomic.h

@@ -3,6 +3,18 @@
 
 #if PLATFORM_WINDOWS
 
+FORCE_INLINE i8 atomic8_fetch(struct atomic8 *x) { return (i8)_InterlockedCompareExchange8((char *)&x->_v, 0, 0); }
+FORCE_INLINE i8 atomic8_fetch_set(struct atomic8 *x, i8 e) { return (i8)_InterlockedExchange8((char *)&x->_v, e); }
+FORCE_INLINE i8 atomic8_fetch_test_set(struct atomic8 *x, i8 c, i8 e) { return (i8)_InterlockedCompareExchange8((char *)&x->_v, e, c); }
+FORCE_INLINE i8 atomic8_fetch_xor(struct atomic8 *x, i8 c) { return (i8)_InterlockedXor8((char *)&x->_v, c); }
+FORCE_INLINE i8 atomic8_fetch_add(struct atomic8 *x, i8 a) { return (i8)_InterlockedExchangeAdd8((char *)&x->_v, a); }
+
+FORCE_INLINE i16 atomic16_fetch(struct atomic16 *x) { return (i16)_InterlockedCompareExchange16(&x->_v, 0, 0); }
+FORCE_INLINE i16 atomic16_fetch_set(struct atomic16 *x, i16 e) { return (i16)_InterlockedExchange16(&x->_v, e); }
+FORCE_INLINE i16 atomic16_fetch_test_set(struct atomic16 *x, i16 c, i16 e) { return (i16)_InterlockedCompareExchange16(&x->_v, e, c); }
+FORCE_INLINE i16 atomic16_fetch_xor(struct atomic16 *x, i16 c) { return (i16)_InterlockedXor16(&x->_v, c); }
+FORCE_INLINE i16 atomic16_fetch_add(struct atomic16 *x, i16 a) { return (i16)_InterlockedExchangeAdd16(&x->_v, a); }
+
 FORCE_INLINE i32 atomic32_fetch(struct atomic32 *x) { return (i32)_InterlockedCompareExchange((volatile long *)&x->_v, 0, 0); }
 FORCE_INLINE i32 atomic32_fetch_set(struct atomic32 *x, i32 e) { return (i32)_InterlockedExchange((volatile long *)&x->_v, e); }
 FORCE_INLINE i32 atomic32_fetch_test_set(struct atomic32 *x, i32 c, i32 e) { return (i32)_InterlockedCompareExchange((volatile long *)&x->_v, e, c); }

src/common.h

@@ -277,9 +277,9 @@ void __asan_unpoison_memory_region(void const volatile *add, size_t);
 #define FIELD_SIZEOF(type, field) sizeof(((type *)0)->field)
 
 #if COMPILER_MSVC && !defined _CRT_USE_BUILTIN_OFFSETOF
-# define FIELD_OFFSETOF(type, field) ((u64)&(((type *)0)->field))
+# define offsetof(type, field) ((u64)&(((type *)0)->field))
 #else
-# define FIELD_OFFSETOF(type, field) __builtin_offsetof(type, field)
+# define offsetof(type, field) __builtin_offsetof(type, field)
 #endif
 
 /* Array */
@@ -391,6 +391,16 @@ GLOBAL const f64 *_f64_nan          = (f64 *)&_f64_nan_u64;
  * Atomics
  * ========================== */
 
+/* NOTE: Must be aligned to 32 bit boundary by user */
+struct atomic8 {
+    volatile i8 _v;
+};
+
+/* NOTE: Must be aligned to 32 bit boundary by user */
+struct atomic16 {
+    volatile i16 _v;
+};
+
 struct atomic32 {
     volatile i32 _v;
 };
@@ -403,6 +413,18 @@ struct atomic64 {
  * Cache-line isolated atomics
  * ========================== */
 
+struct alignas(64) atomic8_padded {
+    struct atomic8 v;
+    u8 _pad[60];
+};
+STATIC_ASSERT(sizeof(struct atomic8_padded) == 64 && alignof(struct atomic8_padded) == 64);
+
+struct alignas(64) atomic16_padded {
+    struct atomic16 v;
+    u8 _pad[60];
+};
+STATIC_ASSERT(sizeof(struct atomic16_padded) == 64 && alignof(struct atomic16_padded) == 64);
+
 struct alignas(64) atomic32_padded {
     struct atomic32 v;
     u8 _pad[60];

src/sys_win32.c

@@ -162,9 +162,10 @@ struct alignas(64) fiber {
     /* ==================================================== */
     char *name_cstr;                            /* 08 bytes */
     /* ==================================================== */
+    struct atomic16 wake_lock;                  /* 02 bytes (aligned) */
     i16 id;                                     /* 02 bytes */
     i16 parent_id;                              /* 02 bytes */
-    struct atomic32 wake_lock;                /* 04 bytes */
+    i16 can_yield;                              /* 02 bytes */
     /* ==================================================== */
     u64 wait_addr;                              /* 08 bytes */
     /* ==================================================== */
@@ -204,6 +205,7 @@ struct alignas(64) fiber {
 };
 STATIC_ASSERT(sizeof(struct fiber) == 128);  /* Padding validation (increase if necessary) */
 STATIC_ASSERT(alignof(struct fiber) == 64);  /* Avoid false sharing */
+STATIC_ASSERT(offsetof(struct fiber, wake_lock) % 4 == 0);  /* Atomic must be aligned */
 STATIC_ASSERT(SYS_MAX_FIBERS < I16_MAX);  /* Max fibers should fit in fiber id */
 
 struct alignas(64) worker_ctx {
@@ -392,8 +394,7 @@ i64 sys_current_scheduler_period_ns(void)
 void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns)
 {
     struct fiber *fiber = fiber_from_id(sys_current_fiber_id());
-    i16 parent_fiber_id = fiber->parent_id;
+    if (fiber->can_yield) {
-    if (parent_fiber_id > 0) {
         *fiber->yield_param = (struct yield_param) {
             .kind = YIELD_KIND_WAIT,
             .wait = {
@@ -403,8 +404,8 @@ void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns)
                 .timeout_ns = timeout_ns
             }
         };
-        struct fiber *parent_fiber = fiber_from_id(parent_fiber_id);
+        ASSERT(fiber->parent_id != 0);
-        job_fiber_yield(fiber, parent_fiber);
+        job_fiber_yield(fiber, fiber_from_id(fiber->parent_id));
     } else {
         i32 timeout_ms = 0;
         if (timeout_ns == I64_MAX) {
@@ -564,7 +565,7 @@ INTERNAL void wake_fibers_locked(i32 num_fibers, struct fiber **fibers)
                 queue->first = info;
             }
             queue->last = info;
-            atomic32_fetch_set(&fiber->wake_lock, 0);
+            atomic16_fetch_set(&fiber->wake_lock, 0);
         }
         tm_unlock(&queue->lock);
     }
@@ -611,7 +612,7 @@ INTERNAL void wake_address(void *addr, i32 count)
             if (wait_addr_list) {
                 fibers = arena_push_array_no_zero(scratch.arena, struct fiber *, wait_addr_list->num_waiters);
                 for (struct fiber *fiber = fiber_from_id(wait_addr_list->first_waiter); fiber && num_fibers < count; fiber = fiber_from_id(fiber->next_addr_waiter)) {
-                    if (atomic32_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) {
+                    if (atomic16_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) {
                         fibers[num_fibers] = fiber;
                         ++num_fibers;
                     }
@@ -660,7 +661,7 @@ INTERNAL void wake_time(u64 time)
                 /* Set waiter wake status & build fibers list */
                 fibers = arena_push_array_no_zero(scratch.arena, struct fiber *, wait_time_list->num_waiters);
                 for (struct fiber *fiber = fiber_from_id(wait_time_list->first_waiter); fiber; fiber = fiber_from_id(fiber->next_time_waiter)) {
-                    if (atomic32_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) {
+                    if (atomic16_fetch_test_set(&fiber->wake_lock, 0, 1) == 0) {
                         fibers[num_fibers] = fiber;
                         ++num_fibers;
                     }
@@ -753,12 +754,15 @@ INTERNAL struct fiber *fiber_alloc(enum fiber_kind kind)
         /* Init win32 fiber */
         if (kind == FIBER_KIND_JOB_WORKER) {
             fiber->addr = CreateFiber(FIBER_STACK_SIZE, job_fiber_entry, (void *)(i64)fiber_id);
+            fiber->can_yield = 1;
         } else {
             fiber->addr = ConvertThreadToFiber((void *)(i64)fiber_id);
+            fiber->can_yield = 0;
         }
     }
-    MEMZERO_STRUCT(&fiber->wait_addr);
+    MEMZERO_STRUCT(&fiber->wake_lock);
-    MEMZERO_STRUCT(&fiber->wait_time);
+    fiber->wait_addr = 0;
+    fiber->wait_time = 0;
     fiber->prev_addr_waiter = 0;
     fiber->next_addr_waiter = 0;
     fiber->prev_time_waiter = 0;