////////////////////////////////////////////////////////////
//~ Mutex

//- Exclusive mutex lock
Lock LockSpinE(Mutex *m, i32 spin)
{
  b32 locked = 0;
  i32 spin_cnt = 0;
  while (!locked)
  {
    ++spin_cnt;
    u32 v = Atomic32FetchTestSet(&m->v, 0, 0x80000000);
    if (v == 0)
    {
      locked = 1;
    }
    else if (v == 0x40000000)
    {
      /* Lock has pending bit set, try to lock */
      u32 swp = Atomic32FetchTestSet(&m->v, v, 0x80000000);
      while (swp != v && swp == 0x40000000)
      {
        v = swp;
        swp = Atomic32FetchTestSet(&m->v, v, 0x80000000);
      }
      v = swp;
      if (v == 0x40000000)
      {
        locked = 1;
      }
    }
    if (!locked && (v & 0xC0000000) == 0)
    {
      /* Lock has shared lockers and no pending waiter, set pending bit */
      u32 swp = Atomic32FetchTestSet(&m->v, v, v | 0x40000000);
      while (swp != v && (swp & 0xC0000000) == 0 && swp != 0)
      {
        v = swp;
        swp = Atomic32FetchTestSet(&m->v, v, v | 0x40000000);
      }
      v = swp;
    }
    /* Pause or wait */
    if (!locked && v != 0 && v != 0x40000000)
    {
      if (spin_cnt < spin)
      {
        _mm_pause();
      }
      else
      {
        FutexYieldNeq(&m->v, &v, 4);
        spin_cnt = 0;
      }
    }
  }

  #if IsRtcEnabled
  Atomic32Set(&m->exclusive_thread_id, ThreadId());
  #endif

  Lock lock = Zi;
  lock.exclusive = 1;
  lock.mutex = m;
  return lock;
}

//- Shared mutex lock
Lock LockSpinS(Mutex *m, i32 spin)
{
  b32 locked = 0;
  i32 spin_cnt = 0;
  while (!locked)
  {
    ++spin_cnt;
    u32 v = Atomic32Fetch(&m->v);
    while (!locked && (v & 0xC0000000) == 0)
    {
      /* Lock has no exclusive or pending exclusive lock, increment shared count */
      u32 swp = Atomic32FetchTestSet(&m->v, v, v + 1);
      if (v == swp)
      {
        locked = 1;
      }
      else
      {
        v = swp;
      }
    }
    /* Pause or wait */
    if (!locked)
    {
      if (spin_cnt < spin)
      {
        _mm_pause();
      }
      else
      {
        FutexYieldNeq(&m->v, &v, 4);
        spin_cnt = 0;
      }
    }
  }
  Lock lock = Zi;
  lock.mutex = m;
  return lock;
}

//- Mutex lock wrappers
Lock LockE(Mutex *m)
{
  return LockSpinE(m, DefaultMutexSpin);
}

Lock LockS(Mutex *m)
{
  return LockSpinS(m, DefaultMutexSpin);
}

void Unlock(Lock *l)
{
  Mutex *m = l->mutex;
  if (l->exclusive)
  {
    #if IsRtcEnabled
    Atomic32Set(&m->exclusive_thread_id, 0);
    #endif
    Atomic32Set(&m->v, 0);
  }
  else
  {
    Atomic32FetchAdd(&m->v, -1);
  }
  FutexWakeNeq(&m->v);
  ZeroStruct(l);
}

////////////////////////////////////////////////////////////
//~ Condition variable

void YieldOnCv(Cv *cv, Lock *l)
{
  u64 old_wake_gen = Atomic64Fetch(&cv->wake_gen);
  Mutex *mutex = l->mutex;
  b32 exclusive = l->exclusive;
  {
    Unlock(l);
    {
      FutexYieldNeq(&cv->wake_gen, &old_wake_gen, sizeof(old_wake_gen));
    }
    if (exclusive)
    {
      *l = LockE(mutex);
    }
    else
    {
      *l = LockS(mutex);
    }
  }
}

void SignalCv(Cv *cv)
{
  Atomic64FetchAdd(&cv->wake_gen, 1);
  FutexWakeNeq(&cv->wake_gen);
}

////////////////////////////////////////////////////////////
//~ Fence

i64 FetchFence(Fence *fence)
{
  return Atomic64Fetch(&fence->v.v);
}

void SetFence(Fence *fence, i64 x)
{
  Atomic64Set(&fence->v.v, x);
  FutexWakeGte(&fence->v.v);
}

i64 FetchSetFence(Fence *fence, i64 x)
{
  i64 fetch = Atomic64FetchSet(&fence->v.v, x);
  FutexWakeGte(&fence->v.v);
  return fetch;
}

i64 FetchAddFence(Fence *fence, i64 x)
{
  i64 fetch = Atomic64FetchAdd(&fence->v.v, x);
  FutexWakeGte(&fence->v.v);
  return fetch;
}

i64 YieldOnFence(Fence *fence, i64 target)
{
  i64 v = Atomic64Fetch(&fence->v.v);
  while (v < target)
  {
    FutexYieldGte(&fence->v.v, &v, sizeof(v));
    v = Atomic64Fetch(&fence->v.v);
  }
  return v;
}