release: v0.4.4 — SSTable 编码修复（UTF-8 字节估算 + u32 长度字段 + v1/v2 双格式兼容），大内容不再崩溃

dist/metona-sqlark.js

@@ -36,7 +36,7 @@
     // ---------------------------------------------------------------------------
     //  版本
     // ---------------------------------------------------------------------------
-    const VERSION = '0.4.3';
+    const VERSION = '0.4.4';
 
     /**
      * metona-sqlark Shared WHERE Matcher — 统一的条件匹配逻辑
@@ -2715,7 +2715,15 @@
      *
      * 将排序后的 key-value 数据写入 SSTable 格式。
      *
-     * SSTable 文件布局:
+     * v0.4.4 格式 v2（magic "SSTC"）修复：
+     * - 块大小估算改用 UTF-8 字节长度（TextEncoder 预编码），
+     *   此前用字符串 .length（UTF-16 码元）估算而实际写入 UTF-8 字节，
+     *   中文内容（1 字 3 字节）导致缓冲区低估 → 写入越界崩溃
+     * - keyLen/valueLen 从 u16 升级为 u32（此前 >64KB 的 value 长度被截断，
+     *   线性格式整体错乱）
+     * - 大 value 单条独立成块（切分逻辑基于字节估算）
+     *
+     * SSTable 文件布局 (v2):
      * ┌──────────────────────────────────────────────┐
      * │  Data Block 0                               │
      * │  Data Block 1                               │
@@ -2729,58 +2737,56 @@
      * │    - bloom_size    (u32)                    │
      * │    - bloom_hash_count (u32)                 │
      * │    - entry_count   (u32)                    │
-     * │    - magic_number  (u32, 0x53535442 ="SSTB")│
+     * │    - magic_number  (u32, 0x53535443 ="SSTC")│
      * │    - checksum      (u32)                    │
      * └──────────────────────────────────────────────┘
+     * 数据块条目: entryCount(u32) + [keyLen(u32) + key + valueLen(u32) + value]
+     * 索引块条目: [keyLen(u32) + key + blockOffset(u32) + blockSize(u32)]
      */
-    const SSTABLE_MAGIC$1 = 0x53535442; // "SSTB"
+    /** v1 格式魔数（"SSTB"，u16 长度字段，兼容旧文件读取） */
+    const SSTABLE_MAGIC_V1 = 0x53535442;
+    /** v2 格式魔数（"SSTC"，u32 长度字段 + 字节精确估算，v0.4.4） */
+    const SSTABLE_MAGIC_V2 = 0x53535443;
     const SSTABLE_FOOTER_SIZE = 32;
-    // ---------------------------------------------------------------------------
-    //  SSTableBuilder
-    // ---------------------------------------------------------------------------
     class SSTableBuilder {
         constructor(blockSizeLimit = 4096) {
             this.entries = [];
-            this.currentBlock = [];
-            this.currentBlockStartKey = '';
             this.blockSizeLimit = blockSizeLimit;
         }
         /** 添加一个 key-value 条目（必须按键排序添加） */
         add(key, value) {
-            if (this.currentBlock.length === 0) {
-                this.currentBlockStartKey = key;
-            }
-            this.currentBlock.push([key, value]);
             this.entries.push([key, value]);
-            // 如果当前 Block 达到大小限制，切割
-            const estimated = this.estimateBlockSize();
-            if (estimated >= this.blockSizeLimit && this.currentBlock.length > 1) ;
         }
         /**
-         * 构建 SSTable 文件的二进制数据。
-         * 返回 { data: Uint8Array, indexEntries: IndexEntry[], bloomFilter: BloomFilter }
+         * 构建 SSTable 文件的二进制数据（v2 格式）。
+         * 返回 { data: Uint8Array, indexEntries: IndexEntry[] }
          */
         build() {
-            const blocks = this.splitIntoBlocks();
+            // v0.4.4-fix: 预编码全部条目 — 块大小估算必须基于 UTF-8 字节长度，
+            // 字符串 .length 是 UTF-16 码元（中文 1 字 3 字节 vs 1 码元）→ 缓冲区低估越界
+            const encoder = new TextEncoder();
+            const encoded = this.entries.map(([key, value]) => ({
+                key,
+                keyBytes: encoder.encode(key),
+                valueBytes: encoder.encode(JSON.stringify(value)),
+            }));
+            const blocks = this.splitIntoBlocks(encoded);
             const bloomFilter = new BloomFilter(this.entries.length);
-            // 预计算总大小
+            // 预计算总大小（字节）
             let totalSize = 0;
             const blockOffsets = [];
             for (const block of blocks) {
                 blockOffsets.push(totalSize);
-                const blockSize = this.computeBlockSize(block);
-                totalSize += blockSize;
+                totalSize += this.computeBlockSize(block);
             }
-            // 索引块
+            // 索引块（块内最后一个 key 作为索引键）
             const indexEntries = [];
             for (let i = 0; i < blocks.length; i++) {
                 const block = blocks[i];
-                const lastKey = block[block.length - 1][0];
-                const blockSize = this.computeBlockSize(block);
                 indexEntries.push({
-                    key: lastKey,
+                    key: block[block.length - 1].key,
                     blockOffset: blockOffsets[i],
-                    blockSize,
+                    blockSize: this.computeBlockSize(block),
                 });
             }
             const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
@@ -2811,7 +2817,7 @@
             view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
             view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
             view.setUint32(footerOffset + 20, this.entries.length, false);
-            view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
+            view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false);
             view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
             return {
                 sstableData: new Uint8Array(buf),
@@ -2825,12 +2831,14 @@
         // -----------------------------------------------------------------------
         //  内部
         // -----------------------------------------------------------------------
-        splitIntoBlocks() {
+        /** 按 UTF-8 字节大小切分数据块；大 value 单条独立成块 */
+        splitIntoBlocks(encoded) {
             const blocks = [];
             let current = [];
-            for (const entry of this.entries) {
+            for (const entry of encoded) {
                 current.push(entry);
-                if (this.estimateBlockSizeFromEntries(current) >= this.blockSizeLimit && current.length > 1) {
+                // v0.4.4-fix: 基于字节估算；单条超大条目（length===1）独立成块不强行切分
+                if (this.computeBlockSize(current) >= this.blockSizeLimit && current.length > 1) {
                     blocks.push(current.slice(0, -1));
                     current = [entry];
                 }
@@ -2839,22 +2847,11 @@
                 blocks.push(current);
             return blocks;
         }
-        estimateBlockSize() {
-            return this.estimateBlockSizeFromEntries(this.currentBlock);
-        }
-        estimateBlockSizeFromEntries(entries) {
-            let size = 0;
-            for (const [key, value] of entries) {
-                size += 4 + key.length + JSON.stringify(value).length;
-            }
-            return size;
-        }
+        /** 块字节大小：entryCount(u32) + 每对 [keyLen(u32) + key + valueLen(u32) + value] */
         computeBlockSize(block) {
-            // entryCount (u32) + 每对: keyLen(u16) + key + valueLen(u16) + value json
             let size = 4;
-            for (const [key, value] of block) {
-                const json = JSON.stringify(value);
-                size += 2 + key.length + 2 + json.length;
+            for (const e of block) {
+                size += 4 + e.keyBytes.length + 4 + e.valueBytes.length;
             }
             return size;
         }
@@ -2862,32 +2859,30 @@
             // entry count
             view.setUint32(offset, block.length, false);
             offset += 4;
-            for (const [key, value] of block) {
-                const encoder = new TextEncoder();
-                const keyBytes = encoder.encode(key);
-                const valueBytes = encoder.encode(JSON.stringify(value));
-                // key length
-                view.setUint16(offset, keyBytes.length, false);
-                offset += 2;
-                // key
-                new Uint8Array(view.buffer).set(keyBytes, offset);
-                offset += keyBytes.length;
-                // value length
-                view.setUint16(offset, valueBytes.length, false);
-                offset += 2;
-                // value
-                new Uint8Array(view.buffer).set(valueBytes, offset);
-                offset += valueBytes.length;
+            for (const e of block) {
+                // v0.4.4-fix: 长度字段 u32（此前 u16 截断 >64KB 的 value）
+                if (e.keyBytes.length > 0xFFFFFFFF || e.valueBytes.length > 0xFFFFFFFF) {
+                    throw new Error('SSTable entry too large (exceeds u32 length field)');
+                }
+                view.setUint32(offset, e.keyBytes.length, false);
+                offset += 4;
+                new Uint8Array(view.buffer).set(e.keyBytes, offset);
+                offset += e.keyBytes.length;
+                view.setUint32(offset, e.valueBytes.length, false);
+                offset += 4;
+                new Uint8Array(view.buffer).set(e.valueBytes, offset);
+                offset += e.valueBytes.length;
                 // 插入 bloom filter
-                bloomFilter.insert(key);
+                bloomFilter.insert(e.key);
             }
             return offset;
         }
         estimateIndexBlockSize(entries) {
-            // entryCount(u32) + each: keyLen(u16)+key+blockOffset(u32)+blockSize(u32)
+            // entryCount(u32) + each: keyLen(u32)+key+blockOffset(u32)+blockSize(u32)
             let size = 4;
+            const encoder = new TextEncoder();
             for (const entry of entries) {
-                size += 2 + entry.key.length + 8;
+                size += 4 + encoder.encode(entry.key).byteLength + 8;
             }
             return size;
         }
@@ -2897,8 +2892,8 @@
             for (const entry of entries) {
                 const encoder = new TextEncoder();
                 const keyBytes = encoder.encode(entry.key);
-                view.setUint16(offset, keyBytes.length, false);
-                offset += 2;
+                view.setUint32(offset, keyBytes.length, false);
+                offset += 4;
                 new Uint8Array(view.buffer).set(keyBytes, offset);
                 offset += keyBytes.length;
                 view.setUint32(offset, entry.blockOffset, false);
@@ -2913,8 +2908,10 @@
     /**
      * AriaEngine SSTable Reader — 从 SSTable 二进制数据中读取
      * @module engine/aria/index/sstable
+     *
+     * v0.4.4: 支持 v1（"SSTB"，u16 长度字段）与 v2（"SSTC"，u32 长度字段）双格式，
+     * 旧库 v1 文件仍可读（小 value 场景无缺陷），新写入使用 v2。
      */
-    const SSTABLE_MAGIC = 0x53535442; // "SSTB"
     // ---------------------------------------------------------------------------
     //  SSTableReader
     // ---------------------------------------------------------------------------
@@ -2923,11 +2920,20 @@
             this.indexEntries = [];
             this.entryCount = 0;
             this.bloomFilter = null;
+            /** 格式版本：1 = u16 长度字段（旧），2 = u32 长度字段（v0.4.4） */
+            this.format = 2;
             this.data = data;
             this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
             this.meta = meta;
             this.parseFooter();
         }
+        /** 长度字段宽度：v2 = 4 字节 u32，v1 = 2 字节 u16 */
+        lenFieldSize() {
+            return this.format === 2 ? 4 : 2;
+        }
+        readLen(offset) {
+            return this.format === 2 ? this.view.getUint32(offset, false) : this.view.getUint16(offset, false);
+        }
         // -----------------------------------------------------------------------
         //  查询
         // -----------------------------------------------------------------------
@@ -2941,24 +2947,25 @@
                 return null;
             const entry = this.indexEntries[blockIdx];
             const blockData = this.getBlockData(entry);
-            // v0.4.2-fix: 残缺文件（meta 偏移超出实际长度）跳过该块，而非抛 RangeError
+            // v0.4.1-fix: 残缺文件（meta 偏移超出实际长度）跳过该块，而非抛 RangeError
             if (!blockData)
                 return null;
             const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
+            const lenSize = this.lenFieldSize();
             const entryCount = blockView.getUint32(0, false);
             let offset = 4;
             // 顺序扫描 block 内的条目（生产中应二分查找）
             for (let i = 0; i < entryCount; i++) {
-                if (offset + 2 > blockData.byteLength)
+                if (offset + lenSize > blockData.byteLength)
                     break;
-                const keyLen = blockView.getUint16(offset, false);
-                offset += 2;
-                if (offset + keyLen + 2 > blockData.byteLength)
+                const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                offset += lenSize;
+                if (offset + keyLen + lenSize > blockData.byteLength)
                     break;
                 const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
                 offset += keyLen;
-                const valLen = blockView.getUint16(offset, false);
-                offset += 2;
+                const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                offset += lenSize;
                 if (offset + valLen > blockData.byteLength)
                     break;
                 const valBytes = blockData.slice(offset, offset + valLen);
@@ -2982,26 +2989,27 @@
             const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey));
             if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx)
                 return;
+            const lenSize = this.lenFieldSize();
             for (let bi = startBlockIdx; bi <= endBlockIdx && bi >= 0; bi++) {
                 const entry = this.indexEntries[bi];
                 const blockData = this.getBlockData(entry);
-                // v0.4.2-fix: 残缺块跳过（rangeScan 继续后续块，不抛异常）
+                // v0.4.1-fix: 残缺块跳过（rangeScan 继续后续块，不抛异常）
                 if (!blockData)
                     continue;
                 const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
                 const blockEntryCount = blockView.getUint32(0, false);
                 let offset = 4;
                 for (let i = 0; i < blockEntryCount; i++) {
-                    if (offset + 2 > blockData.byteLength)
+                    if (offset + lenSize > blockData.byteLength)
                         break;
-                    const keyLen = blockView.getUint16(offset, false);
-                    offset += 2;
-                    if (offset + keyLen + 2 > blockData.byteLength)
+                    const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                    offset += lenSize;
+                    if (offset + keyLen + lenSize > blockData.byteLength)
                         break;
                     const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
                     offset += keyLen;
-                    const valLen = blockView.getUint16(offset, false);
-                    offset += 2;
+                    const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                    offset += lenSize;
                     if (offset + valLen > blockData.byteLength)
                         break;
                     const valBytes = blockData.slice(offset, offset + valLen);
@@ -3020,25 +3028,26 @@
         }
         /** 扫描所有条目 */
         scanAll(callback) {
+            const lenSize = this.lenFieldSize();
             for (const entry of this.indexEntries) {
                 const blockData = this.getBlockData(entry);
-                // v0.4.2-fix: 残缺块跳过（scanAll 继续后续块，不抛异常）
+                // v0.4.1-fix: 残缺块跳过（scanAll 继续后续块，不抛异常）
                 if (!blockData)
                     continue;
                 const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
                 const blockEntryCount = blockView.getUint32(0, false);
                 let offset = 4;
                 for (let i = 0; i < blockEntryCount; i++) {
-                    if (offset + 2 > blockData.byteLength)
+                    if (offset + lenSize > blockData.byteLength)
                         break;
-                    const keyLen = blockView.getUint16(offset, false);
-                    offset += 2;
-                    if (offset + keyLen + 2 > blockData.byteLength)
+                    const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                    offset += lenSize;
+                    if (offset + keyLen + lenSize > blockData.byteLength)
                         break;
                     const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
                     offset += keyLen;
-                    const valLen = blockView.getUint16(offset, false);
-                    offset += 2;
+                    const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                    offset += lenSize;
                     if (offset + valLen > blockData.byteLength)
                         break;
                     const valBytes = blockData.slice(offset, offset + valLen);
@@ -3069,10 +3078,16 @@
                 throw new Error('SSTable too small: missing footer');
             }
             const footerOffset = this.data.byteLength - 32;
-            // 验证魔数
+            // 验证魔数（v1 "SSTB" / v2 "SSTC"）
             const magic = this.view.getUint32(footerOffset + 24, false);
-            if (magic !== SSTABLE_MAGIC) {
-                throw new Error(`Invalid SSTable magic: expected ${SSTABLE_MAGIC}, got ${magic}`);
+            if (magic === SSTABLE_MAGIC_V1) {
+                this.format = 1;
+            }
+            else if (magic === SSTABLE_MAGIC_V2) {
+                this.format = 2;
+            }
+            else {
+                throw new Error(`Invalid SSTable magic: expected ${SSTABLE_MAGIC_V1} or ${SSTABLE_MAGIC_V2}, got ${magic}`);
             }
             const indexOffset = this.view.getUint32(footerOffset, false);
             const indexSize = this.view.getUint32(footerOffset + 4, false);
@@ -3080,7 +3095,7 @@
             const bloomSize = this.view.getUint32(footerOffset + 12, false);
             const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
             this.entryCount = this.view.getUint32(footerOffset + 20, false);
-            // v0.4.2-fix: 完整性校验 — 索引块必须完全落在文件内，否则视为残缺文件跳过
+            // v0.4.1-fix: 完整性校验 — 索引块必须完全落在文件内，否则视为残缺文件跳过
             if (indexOffset + 4 > this.data.byteLength || indexOffset + indexSize > this.data.byteLength) {
                 return; // 残缺文件：无索引块可读，get/rangeScan 均返回空
             }
@@ -3100,13 +3115,14 @@
         parseIndexBlock(offset, _size) {
             const entryCount = this.view.getUint32(offset, false);
             offset += 4;
+            const lenSize = this.lenFieldSize();
             for (let i = 0; i < entryCount; i++) {
-                // v0.4.2-fix: 索引条目越界（keyLen/blockOffset/blockSize 超过文件长度）时中止解析，
+                // v0.4.1-fix: 索引条目越界（keyLen/blockOffset/blockSize 超过文件长度）时中止解析，
                 // 已解析的有效条目仍可用于查询
-                if (offset + 2 > this.data.byteLength)
+                if (offset + lenSize > this.data.byteLength)
                     break;
-                const keyLen = this.view.getUint16(offset, false);
-                offset += 2;
+                const keyLen = this.readLen(offset);
+                offset += lenSize;
                 if (offset + keyLen + 8 > this.data.byteLength)
                     break;
                 const key = new TextDecoder().decode(this.data.slice(offset, offset + keyLen));
@@ -3122,7 +3138,7 @@
             }
         }
         /**
-         * v0.4.2-fix: 获取索引条目对应的数据块。
+         * v0.4.1-fix: 获取索引条目对应的数据块。
          * 块偏移/大小越界（残缺 SSTable）时返回 null，由调用方跳过而非抛 RangeError。
          */
         getBlockData(entry) {