/** * AriaEngine SSTable Builder — 构建有序字符串表 * @module engine/aria/index/sstable_builder * * 将排序后的 key-value 数据写入 SSTable 格式。 * * v0.4.4 格式 v2(magic "SSTC")修复: * - 块大小估算改用 UTF-8 字节长度(TextEncoder 预编码), * 此前用字符串 .length(UTF-16 码元)估算而实际写入 UTF-8 字节, * 中文内容(1 字 3 字节)导致缓冲区低估 → 写入越界崩溃 * - keyLen/valueLen 从 u16 升级为 u32(此前 >64KB 的 value 长度被截断, * 线性格式整体错乱) * - 大 value 单条独立成块(切分逻辑基于字节估算) * * SSTable 文件布局 (v2): * ┌──────────────────────────────────────────────┐ * │ Data Block 0 │ * │ Data Block 1 │ * │ ... │ * │ Index Block (block offset → key range) │ * │ Bloom Filter │ * │ Footer (32 bytes) │ * │ - index_offset (u32) │ * │ - index_size (u32) │ * │ - bloom_offset (u32) │ * │ - bloom_size (u32) │ * │ - bloom_hash_count (u32) │ * │ - entry_count (u32) │ * │ - magic_number (u32, 0x53535443 ="SSTC")│ * │ - checksum (u32) │ * └──────────────────────────────────────────────┘ * 数据块条目: entryCount(u32) + [keyLen(u32) + key + valueLen(u32) + value] * 索引块条目: [keyLen(u32) + key + blockOffset(u32) + blockSize(u32)] */ import { BloomFilter } from './bloom'; import { DEFAULT_BLOOM_BITS_PER_KEY } from '../types'; import { crc32 } from '../crc32'; import type { IndexEntry } from '../types'; /** v1 格式魔数("SSTB",u16 长度字段,兼容旧文件读取) */ export const SSTABLE_MAGIC_V1 = 0x53535442; /** v2 格式魔数("SSTC",u32 长度字段 + 字节精确估算,v0.4.4) */ export const SSTABLE_MAGIC_V2 = 0x53535443; const SSTABLE_FOOTER_SIZE = 32; // --------------------------------------------------------------------------- // SSTableBuilder // --------------------------------------------------------------------------- interface EncodedEntry { key: string; keyBytes: Uint8Array; valueBytes: Uint8Array; } export class SSTableBuilder { /** Bloom Filter 每 key 位数(由配置透传) */ private bloomBitsPerKey: number; private entries: [string, Record][] = []; private blockSizeLimit: number; /** * @param blockSizeLimit 块大小上限(字节) * @param bloomBitsPerKey Bloom Filter 每 key 位数(v0.8.0 审查修复:此前 * 无论 `bloomFilterBitsPerKey` 配成多少,这里都写死用默认值 —— * 配置项被接受却完全不起作用) */ constructor( blockSizeLimit: number = 4096, bloomBitsPerKey: number = DEFAULT_BLOOM_BITS_PER_KEY, ) { this.blockSizeLimit = blockSizeLimit; this.bloomBitsPerKey = Number.isFinite(bloomBitsPerKey) && bloomBitsPerKey > 0 ? Math.floor(bloomBitsPerKey) : DEFAULT_BLOOM_BITS_PER_KEY; } /** 添加一个 key-value 条目(必须按键排序添加) */ add(key: string, value: Record): void { this.entries.push([key, value]); } /** * 构建 SSTable 文件的二进制数据(v2 格式)。 * 返回 { data: Uint8Array, indexEntries: IndexEntry[] } */ build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } { // v0.4.4-fix: 预编码全部条目 — 块大小估算必须基于 UTF-8 字节长度, // 字符串 .length 是 UTF-16 码元(中文 1 字 3 字节 vs 1 码元)→ 缓冲区低估越界 const encoder = new TextEncoder(); const encoded: EncodedEntry[] = this.entries.map(([key, value]) => ({ key, keyBytes: encoder.encode(key), valueBytes: encoder.encode(JSON.stringify(value)), })); const blocks = this.splitIntoBlocks(encoded); const bloomFilter = new BloomFilter(this.entries.length, this.bloomBitsPerKey); // 预计算总大小(字节) let totalSize = 0; const blockOffsets: number[] = []; for (const block of blocks) { blockOffsets.push(totalSize); totalSize += this.computeBlockSize(block); } // 索引块(块内最后一个 key 作为索引键) const indexEntries: IndexEntry[] = []; for (let i = 0; i < blocks.length; i++) { const block = blocks[i]; indexEntries.push({ key: block[block.length - 1].key, blockOffset: blockOffsets[i], blockSize: this.computeBlockSize(block), }); } const indexBlockSize = this.estimateIndexBlockSize(indexEntries); // 序列化 bloom filter 以获取其大小 const bloomData = bloomFilter.serialize(); const bloomSize = bloomData.byteLength; // 写入到 buffer(包含 bloom block) const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE; const buf = new ArrayBuffer(finalSize); const view = new DataView(buf); let offset = 0; // ---- Data Blocks ---- for (const block of blocks) { offset = this.writeDataBlock(view, offset, block, bloomFilter); } // ---- Index Block ---- const indexOffset = offset; offset = this.writeIndexBlock(view, offset, indexEntries); // ---- Bloom Filter Block ---- const bloomOffset = offset; new Uint8Array(view.buffer).set(bloomData, offset); offset += bloomSize; // ---- Footer ---- const footerOffset = offset; view.setUint32(footerOffset, indexOffset, false); // index_offset view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false); view.setUint32(footerOffset + 20, this.entries.length, false); view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false); // checksum 字段先写 0,全部字节就绪后计算整文件 CRC32 再回填 view.setUint32(footerOffset + 28, 0, false); // v0.4.5: 真实 CRC-32 校验和 — 覆盖除自身(最后 4 字节)外的全部内容。 // checksum 永远非 0(计算结果为 0 时用 1 代替),读取端以 0 识别旧版无校验文件 const all = new Uint8Array(buf); let checksum = crc32(all.subarray(0, all.byteLength - 4)); if (checksum === 0) checksum = 1; view.setUint32(footerOffset + 28, checksum, false); return { sstableData: new Uint8Array(buf), indexEntries, }; } /** 获取条目数 */ getEntryCount(): number { return this.entries.length; } // ----------------------------------------------------------------------- // 内部 // ----------------------------------------------------------------------- /** 按 UTF-8 字节大小切分数据块;大 value 单条独立成块 */ private splitIntoBlocks(encoded: EncodedEntry[]): EncodedEntry[][] { const blocks: EncodedEntry[][] = []; let current: EncodedEntry[] = []; for (const entry of encoded) { current.push(entry); // v0.4.4-fix: 基于字节估算;单条超大条目(length===1)独立成块不强行切分 if (this.computeBlockSize(current) >= this.blockSizeLimit && current.length > 1) { blocks.push(current.slice(0, -1)); current = [entry]; } } if (current.length > 0) blocks.push(current); return blocks; } /** 块字节大小:entryCount(u32) + 每对 [keyLen(u32) + key + valueLen(u32) + value] */ private computeBlockSize(block: EncodedEntry[]): number { let size = 4; for (const e of block) { size += 4 + e.keyBytes.length + 4 + e.valueBytes.length; } return size; } private writeDataBlock( view: DataView, offset: number, block: EncodedEntry[], bloomFilter: BloomFilter, ): number { // entry count view.setUint32(offset, block.length, false); offset += 4; for (const e of block) { // v0.4.4-fix: 长度字段 u32(此前 u16 截断 >64KB 的 value) if (e.keyBytes.length > 0xFFFFFFFF || e.valueBytes.length > 0xFFFFFFFF) { throw new Error('SSTable entry too large (exceeds u32 length field)'); } view.setUint32(offset, e.keyBytes.length, false); offset += 4; new Uint8Array(view.buffer).set(e.keyBytes, offset); offset += e.keyBytes.length; view.setUint32(offset, e.valueBytes.length, false); offset += 4; new Uint8Array(view.buffer).set(e.valueBytes, offset); offset += e.valueBytes.length; // 插入 bloom filter bloomFilter.insert(e.key); } return offset; } private estimateIndexBlockSize(entries: IndexEntry[]): number { // entryCount(u32) + each: keyLen(u32)+key+blockOffset(u32)+blockSize(u32) let size = 4; const encoder = new TextEncoder(); for (const entry of entries) { size += 4 + encoder.encode(entry.key).byteLength + 8; } return size; } private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number { view.setUint32(offset, entries.length, false); offset += 4; for (const entry of entries) { const encoder = new TextEncoder(); const keyBytes = encoder.encode(entry.key); view.setUint32(offset, keyBytes.length, false); offset += 4; new Uint8Array(view.buffer).set(keyBytes, offset); offset += keyBytes.length; view.setUint32(offset, entry.blockOffset, false); offset += 4; view.setUint32(offset, entry.blockSize, false); offset += 4; } return offset; } }