/* TODO: Safety check that functions taking byte length can't overflow bit conversion (log2(num_bytes) > 61) */ //////////////////////////////////////////////////////////// //~ Buff management BB_Buff BB_AcquireBuff(u64 arena_reserve) { BB_Buff result = ZI; result.arena = AcquireArena(arena_reserve); result.is_backed_by_arena = 1; return result; } void BB_ReleaseBuff(BB_Buff *bb) { /* Only arena bitbuffs need to be released */ if (bb->is_backed_by_arena) { ReleaseArena(bb->arena); } } BB_Buff BB_BuffFromString(String s) { BB_Buff result = ZI; result.fixed_buffer = s; return result; } //////////////////////////////////////////////////////////// //~ Writer management BB_Writer BB_WriterFromBuff(BB_Buff *bb) { BB_Writer result = ZI; result.bb = bb; if (bb->is_backed_by_arena) { result.base = ArenaBase(bb->arena); } else { result.base = bb->fixed_buffer.text; } result.cur_bit = 0; #if BITBUFF_DEBUG result.debug_enabled = 1; #endif return result; } /* Use this when writing external formats that will not verify bitbuff debug symbols / magic numbers */ BB_Writer BB_WriterFromBuffNoDebug(BB_Buff *bb) { BB_Writer result = BB_WriterFromBuff(bb); #if BITBUFF_DEBUG result.debug_enabled = 0; #endif return result; } /* FIXME: Handle overflowed bw */ u64 BB_GetNumBitsWritten(BB_Writer *bw) { return bw->cur_bit; } /* FIXME: Handle overflowed bw */ u64 BB_GetNumBytesWritten(BB_Writer *bw) { return (bw->cur_bit + 7) >> 3; } /* FIXME: Handle overflowed bw */ String BB_GetWritten(Arena *arena, BB_Writer *bw) { String result = ZI; result.len = (bw->cur_bit + 7) >> 3; result.text = PushStructsNoZero(arena, u8, result.len); CopyBytes(result.text, bw->base, result.len); return result; } /* FIXME: Handle overflowed bw */ u8 *BB_GetWrittenRaw(BB_Writer *bw) { return bw->base; } /* Returns 1 if num_bits would cause the writer to overflow its fixed buffer size (if writer is not backed by a dynamic arena bitbuff) */ b32 BB_CheckWriterOverflowBits(BB_Writer *bw, u64 num_bits) { b32 result = 0; BB_Buff *bb = bw->bb; if (bw->overflowed) { result = 1; } else { u64 bytes_needed = (bw->cur_bit + num_bits + 7) >> 3; if (bb->is_backed_by_arena) { Arena *arena = bb->arena; if (bytes_needed >= arena->pos) { /* Grow arena */ u64 push_size = (((bytes_needed - arena->pos) / BB_WriterOverflowArenaPushSize) + 1) * BB_WriterOverflowArenaPushSize; PushStructsNoZero(arena, u8, push_size); } } else { u64 max_len = bb->fixed_buffer.len; if (bytes_needed > max_len) { /* Writer overflowed fixed buffer */ Assert(0); result = 1; bw->cur_bit = max_len << 3; bw->overflowed = 1; } } } return result; } //////////////////////////////////////////////////////////// //~ Align writer /* Align the pos to the next byte */ void BB_WriteAlignToNextByte(BB_Writer *bw) { #if BITBUFF_DEBUG if ((bw->cur_bit & 7) != 0) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_AlignNextByte, 0); } #endif bw->cur_bit += (8 - (bw->cur_bit & 7)) & 7; } void BB_WriteAlignBytes(BB_Writer *bw, u64 align) { BB_WriteAlignToNextByte(bw); BB_WriteDebugMagic(bw, BB_DebugMagicKind_AlignBytes, align); if (align > 0) { u64 new_pos = (bw->cur_bit >> 3); new_pos += (align - 1); new_pos -= new_pos % align; if (BB_CheckWriterOverflowBits(bw, new_pos << 3)) { return; } bw->cur_bit = new_pos << 3; } } //////////////////////////////////////////////////////////// //~ Write bits void BB_WriteUBitsNoMagic(BB_Writer *bw, u64 value, u8 num_bits) { Assert(num_bits > 0 && (num_bits == 64 || value <= ~(U64Max << num_bits))); /* Bit count must be able to hold value */ if (BB_CheckWriterOverflowBits(bw, num_bits)) { return; } u8 offset = bw->cur_bit & 7; if (offset != 0) { /* Write unaligned bits */ u8 *at = bw->base + (bw->cur_bit >> 3); u8 num_mix_bits = MinU8((8 - offset), num_bits); u8 mix_byte = (u8)((value & ((1 << num_mix_bits) - 1)) << offset); *at |= mix_byte; value >>= num_mix_bits; num_bits -= num_mix_bits; bw->cur_bit += num_mix_bits; } /* cur_bit is now aligned to byte */ u8 *at = bw->base + (bw->cur_bit >> 3); u8 num_bytes = (num_bits + 7) >> 3; CopyBytes(at, &value, num_bytes); bw->cur_bit += num_bits; } void BB_WriteUBits(BB_Writer *bw, u64 value, u8 num_bits) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_UBits, num_bits); BB_WriteUBitsNoMagic(bw, value, num_bits); } void BB_WriteIBits(BB_Writer *bw, i64 value, u8 num_bits) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_IBits, num_bits); u64 ubits; if (value >= 0) { ubits = value; } else { ubits = BB_TwosComplimentFromUint(-value, num_bits); } BB_WriteUBits(bw, ubits, num_bits); } /* Returns written bit to make writing delta encoding logic cleaner */ b32 BB_WriteBit(BB_Writer *bw, u8 value) { BB_WriteUBits(bw, value, 1); return value; } //////////////////////////////////////////////////////////// //~ Write variable length integers /* Writes a variable length unsigned integer. * Value is written in chunks of 7 bits w/ 8th bit signaling continuation. */ void BB_WriteUV(BB_Writer *bw, u64 value) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_UV, 0); while (value > 0x7F) { u8 cont_byte = 0x80 | (value & 0x7F); BB_WriteUBits(bw, cont_byte, 8); value >>= 7; } BB_WriteUBits(bw, value, 8); } /* Writes a variable length signed integer. * Similar to BB_WriteUV, except the 7th bit of the first byte is a sign bit * indicating that the value is stored in twos compliment. */ void BB_WriteIV(BB_Writer *bw, i64 value) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_IV, 0); u8 sign_bit; u64 tc; if (value >= 0) { sign_bit = 0; tc = value; } else { sign_bit = 1; u64 unsigned_value = -value; u8 num_bits = 6; unsigned_value >>= 6; while (unsigned_value > 0) { num_bits += 7; unsigned_value >>= 7; } num_bits = MinU8(num_bits, 64); tc = BB_TwosComplimentFromUint(-value, num_bits); } /* First byte contains not just cont bit, but sign bit as well. */ u8 first_byte = (tc & 0x3F); tc >>= 6; first_byte |= (tc > 0) << 7; /* Cont bit */ first_byte |= sign_bit << 6; /* Sign bit */ BB_WriteUBits(bw, first_byte, 8); if (tc > 0) { while (tc > 0x7F) { u8 cont_byte = 0x80 | (tc & 0x7F); BB_WriteUBits(bw, cont_byte, 8); tc >>= 7; } BB_WriteUBits(bw, tc, 8); } } //////////////////////////////////////////////////////////// //~ Write floating point void BB_WriteF32(BB_Writer *bw, f32 value) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_F32, 0); BB_WriteUBits(bw, *(u32 *)&value, 32); } void BB_WriteF64(BB_Writer *bw, f64 value) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_F64, 0); BB_WriteUBits(bw, *(u64 *)&value, 64); } //////////////////////////////////////////////////////////// //~ Write Uid void BB_WriteUid(BB_Writer *bw, Uid value) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_Uid, 128); BB_WriteUBits(bw, value.hi, 64); BB_WriteUBits(bw, value.lo, 64); } //////////////////////////////////////////////////////////// //~ Write raw data void BB_WriteString(BB_Writer *bw, String s) { BB_WriteDebugMagic(bw, BB_DebugMagicKind_String, 0); BB_WriteUV(bw, s.len); BB_WriteBytes(bw, s); } void BB_WriteBytes(BB_Writer *bw, String bytes) { /* Align start of bytes */ BB_WriteAlignToNextByte(bw); u64 num_bits = bytes.len << 3; if (BB_CheckWriterOverflowBits(bw, num_bits)) { return; } u8 *at = bw->base + (bw->cur_bit >> 3); CopyBytes(at, bytes.text, bytes.len); bw->cur_bit += num_bits; } void BB_WriteSeekBytes(BB_Writer *bw, u64 num_bytes) { BB_WriteAlignToNextByte(bw); u64 num_bits = num_bytes << 3; if (BB_CheckWriterOverflowBits(bw, num_bits)) { return; } bw->cur_bit += num_bits; } //////////////////////////////////////////////////////////// //~ Writer debug #if BITBUFF_DEBUG void BB_WriteDebugMarker(BB_Writer *bw, String name) { bw->cur_bit += (8 - (bw->cur_bit & 7)) & 7; for (u64 i = 0; i < name.len; ++i) { BB_WriteUBitsNoMagic(bw, name.text[i], 8); } } void BB_WriteDebugMagic(BB_Writer *bw, BB_DebugMagicKind magic, u8 num_bits) { if (bw->debug_enabled) { if (BB_CheckWriterOverflowBits(bw, 24)) { return; } u64 magic_ubits = (u64)magic | ((u64)num_bits << 16); BB_WriteUBitsNoMagic(bw, magic_ubits, 24); } } #endif //////////////////////////////////////////////////////////// //~ Reader management BB_Reader BB_ReaderFromBuff(BB_Buff *bb) { BB_Reader result = ZI; if (!bb->is_backed_by_arena) { result.base = bb->fixed_buffer.text; result.base_len = bb->fixed_buffer.len; } else { Arena *arena = bb->arena; result.base = ArenaBase(arena); result.base_len = arena->pos; } result.cur_bit = 0; #if BITBUFF_DEBUG result.debug_enabled = 1; #endif return result; } /* Use this when reading from external formats that will not contain bitbuff debug symbols / magic numbers */ BB_Reader BB_ReaderFromBuffNoDebug(BB_Buff *bb) { BB_Reader result = BB_ReaderFromBuff(bb); #if BITBUFF_DEBUG result.debug_enabled = 0; #endif return result; } /* Returns the number of bits read from the bitbuff */ /* FIXME: Handle overflowed br */ u64 BB_GetCurrentReaderBit(BB_Reader *br) { return br->cur_bit; } /* Returns the number of *full* bytes read from the bitbuff */ /* FIXME: Handle overflowed br */ u64 BB_GetCurrentReaderByte(BB_Reader *br) { return br->cur_bit >> 3; } /* Returns the number of bits left until the bitbuff overflows */ /* FIXME: Handle overflowed br */ u64 BB_NumBitsRemaining(BB_Reader *br) { return (br->base_len << 3) - br->cur_bit; } /* Returns the number of *full* bytes left until the bitbuff overflows */ /* FIXME: Handle overflowed br */ u64 BB_NumBytesRemaining(BB_Reader *br) { return br->base_len - (br->cur_bit >> 3); } b32 BB_CheckReaderOverflowBits(BB_Reader *br, u64 num_bits) { b32 result = 0; if (br->overflowed) { result = 1; } else { u64 bits_needed = br->cur_bit + num_bits; u64 base_len_bits = br->base_len << 3; if (bits_needed > base_len_bits) { /* Tried to read past bitbuff memory */ Assert(0); result = 1; br->cur_bit = base_len_bits; br->overflowed = 1; } } return result; } //////////////////////////////////////////////////////////// //~ Align reader /* Align the pos to the next byte */ void BB_ReadAlignToNextByte(BB_Reader *br) { #if BITBUFF_DEBUG if ((br->cur_bit & 7) != 0) { BB_ReadDebugMagic(br, BB_DebugMagicKind_AlignNextByte, 0); } #endif br->cur_bit += (8 - (br->cur_bit & 7)) & 7; } void BB_ReadAlignBytes(BB_Reader *br, u64 align) { BB_ReadAlignToNextByte(br); BB_ReadDebugMagic(br, BB_DebugMagicKind_AlignBytes, align); if (align > 0) { u64 new_pos = (br->cur_bit >> 3); new_pos += (align - 1); new_pos -= new_pos % align; if (BB_CheckReaderOverflowBits(br, new_pos << 3)) { return; } br->cur_bit = new_pos << 3; } } //////////////////////////////////////////////////////////// //~ Read bits u64 BB_ReadUBitsNoMagic(BB_Reader *br, u8 num_bits) { if (BB_CheckReaderOverflowBits(br, num_bits)) { return 0; } u64 result = 0; u8 offset = br->cur_bit & 7; u8 num_trailing_bits = 0; if (offset) { u8 *at = br->base + (br->cur_bit >> 3); num_trailing_bits = MinU8(8 - offset, num_bits); u8 mix_byte = *at; mix_byte >>= offset; mix_byte &= (1 << num_trailing_bits) - 1; result = mix_byte; num_bits -= num_trailing_bits; br->cur_bit += num_trailing_bits; } /* cur_bit is now aligned to byte */ u8 *at = br->base + (br->cur_bit >> 3); u8 num_bytes = (num_bits + 7) >> 3; u64 tmp = 0; CopyBytes(&tmp, at, num_bytes); u64 mask = U64Max; if (num_bits < 64) { mask = ~(U64Max << num_bits); } tmp &= mask; result |= tmp << num_trailing_bits; br->cur_bit += num_bits; return result; } u64 BB_ReadUBits(BB_Reader *br, u8 num_bits) { BB_ReadDebugMagic(br, BB_DebugMagicKind_UBits, num_bits); return BB_ReadUBitsNoMagic(br, num_bits); } i64 BB_ReadIBits(BB_Reader *br, u8 num_bits) { Assert(num_bits > 1); BB_ReadDebugMagic(br, BB_DebugMagicKind_IBits, num_bits); u64 tc = BB_ReadUBits(br, num_bits); return BB_IntFromTwosCompliment(tc, num_bits); } u8 BB_ReadBit(BB_Reader *br) { return BB_ReadUBits(br, 1); } //////////////////////////////////////////////////////////// //~ Read variable length integer /* Read a variable length unsigned integer. * See BB_WriteUV for details. */ u64 BB_ReadUV(BB_Reader *br) { BB_ReadDebugMagic(br, BB_DebugMagicKind_UV, 0); u64 result = 0; for (u64 i = 0; i <= 9; ++i) { u64 part = BB_ReadUBits(br, 8); u8 is_last_part = part <= 0x7F; result |= (part & 0x7F) << (i * 7); if (is_last_part) { break; } } return result; } /* Read a variable length signed integer. * See BB_WriteIV for details. */ i64 BB_ReadIV(BB_Reader *br) { BB_ReadDebugMagic(br, BB_DebugMagicKind_IV, 0); u8 first_byte = BB_ReadUBits(br, 8); u8 cont_bit = first_byte & 0x80; u8 sign_bit = first_byte & 0x40; u8 num_bits = 6; u64 tc = first_byte & 0x3F; if (cont_bit) { for (u64 i = 0; i <= 9; ++i) { u64 part = BB_ReadUBits(br, 8); u8 is_last_part = part <= 0x7F; tc |= (part & 0x7F) << num_bits; num_bits += 7; if (is_last_part) { break; } } } num_bits = MinU8(num_bits, 64); i64 result; if (sign_bit) { /* Sign bit is 1, indicating result is stored in twos compliment */ result = BB_IntFromTwosCompliment(tc, num_bits); } else { result = (i64)tc; } return result; } //////////////////////////////////////////////////////////// //~ Read floating point f32 BB_ReadF32(BB_Reader *br) { BB_ReadDebugMagic(br, BB_DebugMagicKind_F32, 0); u32 ubits = BB_ReadUBits(br, 32); return *(f32 *)&ubits; } f64 BB_ReadF64(BB_Reader *br) { BB_ReadDebugMagic(br, BB_DebugMagicKind_F64, 0); u64 ubits = BB_ReadUBits(br, 64); return *(f64 *)&ubits; } //////////////////////////////////////////////////////////// //~ Read Uid Uid BB_ReadUid(BB_Reader *br) { BB_ReadDebugMagic(br, BB_DebugMagicKind_Uid, 128); u64 hi = BB_ReadUBits(br, 64); u64 lo = BB_ReadUBits(br, 64); return UID(hi, lo); } //////////////////////////////////////////////////////////// //~ Read raw data String BB_ReadString(Arena *arena, BB_Reader *br) { BB_ReadDebugMagic(br, BB_DebugMagicKind_String, 0); String result = ZI; u64 len = BB_ReadUV(br); u8 *src = BB_ReadBytesRaw(br, len); if (src != 0) { result.len = len; result.text = PushStructsNoZero(arena, u8, len); CopyBytes(result.text, src, len); } return result; } /* Will fill dst with zeroes if bitbuff overflows */ void BB_ReadBytes(BB_Reader *br, String out) { u8 *src = BB_ReadBytesRaw(br, out.len); if (src) { CopyBytes(out.text, src, out.len); } else { ZeroBytes(out.text, out.len); } } /* Will return 0 on bitbuff overflow. Result should be checked. */ u8 *BB_ReadBytesRaw(BB_Reader *br, u64 num_bytes) { BB_ReadAlignToNextByte(br); u64 num_bits = num_bytes << 3; if (BB_CheckReaderOverflowBits(br, num_bits)) { return 0; } u8 *raw = br->base + (br->cur_bit >> 3); br->cur_bit += num_bits; return raw; } void BB_ReadSeekBytes(BB_Reader *br, u64 num_bytes) { BB_ReadAlignToNextByte(br); u64 num_bits = num_bytes << 3; if (BB_CheckReaderOverflowBits(br, num_bits)) { return; } br->cur_bit += num_bits; } void BB_ReadSeekToByte(BB_Reader *br, u64 pos) { u64 cur_byte_pos = br->cur_bit >> 3; if (pos >= cur_byte_pos) { BB_ReadSeekBytes(br, pos - cur_byte_pos); } else { /* Tried to seek byte backwards in reader */ Assert(0); br->overflowed = 1; br->cur_bit = (br->base_len << 3); } } //////////////////////////////////////////////////////////// //~ Reader debug #if BITBUFF_DEBUG void BB_ReadDebugMagic(BB_Reader *br, BB_DebugMagicKind expected_magic, u8 expected_num_bits) { if (br->debug_enabled) { if (BB_CheckReaderOverflowBits(br, 24)) { return; } u64 stored = BB_ReadUBitsNoMagic(br, 24); BB_DebugMagicKind stored_magic = stored & 0xFFFF; u8 stored_num_bits = (stored >> 16) & 0xFF; /* Verify stored magic match */ Assert(expected_magic == stored_magic); /* Verify stored bit count match */ Assert(expected_num_bits == stored_num_bits); } } void BB_ReadDebugMarker(BB_Reader *br, String name) { br->cur_bit += (8 - (br->cur_bit & 7)) & 7; for (u64 i = 0; i < name.len; ++i) { u8 c_stored = BB_ReadUBitsNoMagic(br, 8); u8 c_expected = name.text[i]; Assert(c_expected == c_stored); } } #endif //////////////////////////////////////////////////////////// //~ Utils u64 BB_TwosComplimentFromUint(u64 value, u8 num_bits) { u64 mask = U64Max; if (num_bits < 64) { mask = ~(U64Max << num_bits); } u64 tc = (~value & mask) + 1; tc &= mask; return tc; } i64 BB_IntFromTwosCompliment(u64 tc, u8 num_bits) { u64 msb_mask = (u64)1 << (num_bits - 1); i64 value = -(i64)(tc & msb_mask); value += tc & ~msb_mask; return value; } //////////////////////////////////////////////////////////// //~ Test #if BITBUFF_TEST void BB_Test(void) { TempArena scratch = BeginScratchNoConflict(); u8 kind_ubits = 0; u8 kind_ibits = 1; u8 kind_uv = 2; u8 kind_iv = 3; u8 kind_string = 4; struct test_case_ubits { u64 v; u64 num_bits; }; struct test_case_ibits { i64 v; u64 num_bits; }; struct test_case_uv { u64 v; }; struct test_case_iv { i64 v; }; struct test_case_string { String v; }; struct test_case { u8 kind; union { struct test_case_ubits ubits; struct test_case_ibits ibits; struct test_case_uv uv; struct test_case_iv iv; struct test_case_string s; }; }; struct test_case cases[] = { { kind_ubits, .ubits = { 40, 8 } }, { kind_ubits, .ubits = { 32, 8 } }, { kind_ubits, .ubits = { 100, 7 } }, { kind_ubits, .ubits = { 4, 3 } }, { kind_ubits, .ubits = { 13, 8 } }, { kind_ibits, .ibits = { 0, 8 } }, { kind_ibits, .ibits = { -1, 8 } }, { kind_ibits, .ibits = { -2, 8 } }, { kind_ibits, .ibits = { -3, 8 } }, { kind_ibits, .ibits = { -100, 8 } }, { kind_ibits, .ibits = { -50, 7 } }, { kind_ibits, .ibits = { 50, 7 } }, { kind_ibits, .ibits = { 4, 7 } }, { kind_ibits, .ibits = { 1, 7 } }, { kind_ibits, .ibits = { 3, 3 } }, { kind_ibits, .ibits = { 1, 2 } }, { kind_ibits, .ibits = { 0, 2 } }, { kind_ibits, .ibits = { -1, 2 } }, { kind_uv, .uv = { 0 } }, { kind_uv, .uv = { 100 } }, { kind_uv, .uv = { 10000 } }, { kind_uv, .uv = { 10000000000000 } }, { kind_uv, .uv = { U64Max } }, { kind_iv, .iv = { 0 } }, { kind_iv, .iv = { -1 } }, { kind_iv, .iv = { 10000000000000 } }, { kind_iv, .iv = { -10000000000000 } }, { kind_iv, .iv = { I64Max } }, { kind_iv, .iv = { I64Min } }, { kind_string, .s = { Lit("Hello there! Hope you're doing well.") } }, { kind_ibits, .ibits = { 3, 3 } }, { kind_string, .s = { Lit("Alriiiiiiiiiiiiiiiiiiighty then") } }, { kind_string, .s = { Lit("Alriiiiiiiiiiiiiiiiiiighty then") } }, }; String encoded = ZI; { BB_Buff bb = BB_AcquireBuff(Gibi(64)); BB_Writer bw = BB_WriterFromBuff(&bb); for (u64 i = 0; i < countof(cases); ++i) { struct test_case c = cases[i]; if (c.kind == kind_ubits) { BB_WriteUBits(&bw, c.ubits.v, c.ubits.num_bits); } else if (c.kind == kind_ibits) { BB_WriteIBits(&bw, c.ibits.v, c.ibits.num_bits); } else if (c.kind == kind_uv) { BB_WriteUV(&bw, c.uv.v); } else if (c.kind == kind_iv) { BB_WriteIV(&bw, c.iv.v); } else if (c.kind == kind_string) { BB_WriteString(&bw, c.s.v); } else { Assert(0); } } encoded = BB_GetWritten(scratch.arena, &bw); } { BB_Buff bb = BB_BuffFromString(encoded); BB_Reader br = BB_ReaderFromBuff(&bb); for (u64 i = 0; i < countof(cases); ++i) { struct test_case c = cases[i]; if (c.kind == kind_ubits) { u64 w = c.ubits.v; u64 r = BB_ReadUBits(&br, c.ubits.num_bits); Assert(r == w); } else if (c.kind == kind_ibits) { i64 w = c.ibits.v; i64 r = BB_ReadIBits(&br, c.ubits.num_bits); Assert(r == w); } else if (c.kind == kind_uv) { u64 w = c.uv.v; u64 r = BB_ReadUV(&br); Assert(r == w); } else if (c.kind == kind_iv) { i64 w = c.iv.v; i64 r = BB_ReadIV(&br); Assert(r == w); } else if (c.kind == kind_string) { String w = c.s.v; String r = BB_ReadString(scratch.arena, &br); Assert(EqString(r, w)); } else { Assert(0); } } } EndScratch(scratch); } #endif