release: v0.4.4 — SSTable 编码修复（UTF-8 字节估算 + u32 长度字段 + v1/v2 双格式兼容），大内容不再崩溃

dist/metona-sqlark.esm.js

@@ -30,7 +30,7 @@ class DatabaseError extends Error {
 // ---------------------------------------------------------------------------
 //  版本
 // ---------------------------------------------------------------------------
-const VERSION = '0.4.3';
+const VERSION = '0.4.4';
 
 /**
  * metona-sqlark Shared WHERE Matcher — 统一的条件匹配逻辑
@@ -2709,7 +2709,15 @@ class BloomFilter {
  *
  * 将排序后的 key-value 数据写入 SSTable 格式。
  *
- * SSTable 文件布局:
+ * v0.4.4 格式 v2（magic "SSTC"）修复：
+ * - 块大小估算改用 UTF-8 字节长度（TextEncoder 预编码），
+ *   此前用字符串 .length（UTF-16 码元）估算而实际写入 UTF-8 字节，
+ *   中文内容（1 字 3 字节）导致缓冲区低估 → 写入越界崩溃
+ * - keyLen/valueLen 从 u16 升级为 u32（此前 >64KB 的 value 长度被截断，
+ *   线性格式整体错乱）
+ * - 大 value 单条独立成块（切分逻辑基于字节估算）
+ *
+ * SSTable 文件布局 (v2):
  * ┌──────────────────────────────────────────────┐
  * │  Data Block 0                               │
  * │  Data Block 1                               │
@@ -2723,58 +2731,56 @@ class BloomFilter {
  * │    - bloom_size    (u32)                    │
  * │    - bloom_hash_count (u32)                 │
  * │    - entry_count   (u32)                    │
- * │    - magic_number  (u32, 0x53535442 ="SSTB")│
+ * │    - magic_number  (u32, 0x53535443 ="SSTC")│
  * │    - checksum      (u32)                    │
  * └──────────────────────────────────────────────┘
+ * 数据块条目: entryCount(u32) + [keyLen(u32) + key + valueLen(u32) + value]
+ * 索引块条目: [keyLen(u32) + key + blockOffset(u32) + blockSize(u32)]
  */
-const SSTABLE_MAGIC$1 = 0x53535442; // "SSTB"
+/** v1 格式魔数（"SSTB"，u16 长度字段，兼容旧文件读取） */
+const SSTABLE_MAGIC_V1 = 0x53535442;
+/** v2 格式魔数（"SSTC"，u32 长度字段 + 字节精确估算，v0.4.4） */
+const SSTABLE_MAGIC_V2 = 0x53535443;
 const SSTABLE_FOOTER_SIZE = 32;
-// ---------------------------------------------------------------------------
-//  SSTableBuilder
-// ---------------------------------------------------------------------------
 class SSTableBuilder {
     constructor(blockSizeLimit = 4096) {
         this.entries = [];
-        this.currentBlock = [];
-        this.currentBlockStartKey = '';
         this.blockSizeLimit = blockSizeLimit;
     }
     /** 添加一个 key-value 条目（必须按键排序添加） */
     add(key, value) {
-        if (this.currentBlock.length === 0) {
-            this.currentBlockStartKey = key;
-        }
-        this.currentBlock.push([key, value]);
         this.entries.push([key, value]);
-        // 如果当前 Block 达到大小限制，切割
-        const estimated = this.estimateBlockSize();
-        if (estimated >= this.blockSizeLimit && this.currentBlock.length > 1) ;
     }
     /**
-     * 构建 SSTable 文件的二进制数据。
-     * 返回 { data: Uint8Array, indexEntries: IndexEntry[], bloomFilter: BloomFilter }
+     * 构建 SSTable 文件的二进制数据（v2 格式）。
+     * 返回 { data: Uint8Array, indexEntries: IndexEntry[] }
      */
     build() {
-        const blocks = this.splitIntoBlocks();
+        // v0.4.4-fix: 预编码全部条目 — 块大小估算必须基于 UTF-8 字节长度，
+        // 字符串 .length 是 UTF-16 码元（中文 1 字 3 字节 vs 1 码元）→ 缓冲区低估越界
+        const encoder = new TextEncoder();
+        const encoded = this.entries.map(([key, value]) => ({
+            key,
+            keyBytes: encoder.encode(key),
+            valueBytes: encoder.encode(JSON.stringify(value)),
+        }));
+        const blocks = this.splitIntoBlocks(encoded);
         const bloomFilter = new BloomFilter(this.entries.length);
-        // 预计算总大小
+        // 预计算总大小（字节）
         let totalSize = 0;
         const blockOffsets = [];
         for (const block of blocks) {
             blockOffsets.push(totalSize);
-            const blockSize = this.computeBlockSize(block);
-            totalSize += blockSize;
+            totalSize += this.computeBlockSize(block);
         }
-        // 索引块
+        // 索引块（块内最后一个 key 作为索引键）
         const indexEntries = [];
         for (let i = 0; i < blocks.length; i++) {
             const block = blocks[i];
-            const lastKey = block[block.length - 1][0];
-            const blockSize = this.computeBlockSize(block);
             indexEntries.push({
-                key: lastKey,
+                key: block[block.length - 1].key,
                 blockOffset: blockOffsets[i],
-                blockSize,
+                blockSize: this.computeBlockSize(block),
             });
         }
         const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
@@ -2805,7 +2811,7 @@ class SSTableBuilder {
         view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
         view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
         view.setUint32(footerOffset + 20, this.entries.length, false);
-        view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
+        view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false);
         view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
         return {
             sstableData: new Uint8Array(buf),
@@ -2819,12 +2825,14 @@ class SSTableBuilder {
     // -----------------------------------------------------------------------
     //  内部
     // -----------------------------------------------------------------------
-    splitIntoBlocks() {
+    /** 按 UTF-8 字节大小切分数据块；大 value 单条独立成块 */
+    splitIntoBlocks(encoded) {
         const blocks = [];
         let current = [];
-        for (const entry of this.entries) {
+        for (const entry of encoded) {
             current.push(entry);
-            if (this.estimateBlockSizeFromEntries(current) >= this.blockSizeLimit && current.length > 1) {
+            // v0.4.4-fix: 基于字节估算；单条超大条目（length===1）独立成块不强行切分
+            if (this.computeBlockSize(current) >= this.blockSizeLimit && current.length > 1) {
                 blocks.push(current.slice(0, -1));
                 current = [entry];
             }
@@ -2833,22 +2841,11 @@ class SSTableBuilder {
             blocks.push(current);
         return blocks;
     }
-    estimateBlockSize() {
-        return this.estimateBlockSizeFromEntries(this.currentBlock);
-    }
-    estimateBlockSizeFromEntries(entries) {
-        let size = 0;
-        for (const [key, value] of entries) {
-            size += 4 + key.length + JSON.stringify(value).length;
-        }
-        return size;
-    }
+    /** 块字节大小：entryCount(u32) + 每对 [keyLen(u32) + key + valueLen(u32) + value] */
     computeBlockSize(block) {
-        // entryCount (u32) + 每对: keyLen(u16) + key + valueLen(u16) + value json
         let size = 4;
-        for (const [key, value] of block) {
-            const json = JSON.stringify(value);
-            size += 2 + key.length + 2 + json.length;
+        for (const e of block) {
+            size += 4 + e.keyBytes.length + 4 + e.valueBytes.length;
         }
         return size;
     }
@@ -2856,32 +2853,30 @@ class SSTableBuilder {
         // entry count
         view.setUint32(offset, block.length, false);
         offset += 4;
-        for (const [key, value] of block) {
-            const encoder = new TextEncoder();
-            const keyBytes = encoder.encode(key);
-            const valueBytes = encoder.encode(JSON.stringify(value));
-            // key length
-            view.setUint16(offset, keyBytes.length, false);
-            offset += 2;
-            // key
-            new Uint8Array(view.buffer).set(keyBytes, offset);
-            offset += keyBytes.length;
-            // value length
-            view.setUint16(offset, valueBytes.length, false);
-            offset += 2;
-            // value
-            new Uint8Array(view.buffer).set(valueBytes, offset);
-            offset += valueBytes.length;
+        for (const e of block) {
+            // v0.4.4-fix: 长度字段 u32（此前 u16 截断 >64KB 的 value）
+            if (e.keyBytes.length > 0xFFFFFFFF || e.valueBytes.length > 0xFFFFFFFF) {
+                throw new Error('SSTable entry too large (exceeds u32 length field)');
+            }
+            view.setUint32(offset, e.keyBytes.length, false);
+            offset += 4;
+            new Uint8Array(view.buffer).set(e.keyBytes, offset);
+            offset += e.keyBytes.length;
+            view.setUint32(offset, e.valueBytes.length, false);
+            offset += 4;
+            new Uint8Array(view.buffer).set(e.valueBytes, offset);
+            offset += e.valueBytes.length;
             // 插入 bloom filter
-            bloomFilter.insert(key);
+            bloomFilter.insert(e.key);
         }
         return offset;
     }
     estimateIndexBlockSize(entries) {
-        // entryCount(u32) + each: keyLen(u16)+key+blockOffset(u32)+blockSize(u32)
+        // entryCount(u32) + each: keyLen(u32)+key+blockOffset(u32)+blockSize(u32)
         let size = 4;
+        const encoder = new TextEncoder();
         for (const entry of entries) {
-            size += 2 + entry.key.length + 8;
+            size += 4 + encoder.encode(entry.key).byteLength + 8;
         }
         return size;
     }
@@ -2891,8 +2886,8 @@ class SSTableBuilder {
         for (const entry of entries) {
             const encoder = new TextEncoder();
             const keyBytes = encoder.encode(entry.key);
-            view.setUint16(offset, keyBytes.length, false);
-            offset += 2;
+            view.setUint32(offset, keyBytes.length, false);
+            offset += 4;
             new Uint8Array(view.buffer).set(keyBytes, offset);
             offset += keyBytes.length;
             view.setUint32(offset, entry.blockOffset, false);
@@ -2907,8 +2902,10 @@ class SSTableBuilder {
 /**
  * AriaEngine SSTable Reader — 从 SSTable 二进制数据中读取
  * @module engine/aria/index/sstable
+ *
+ * v0.4.4: 支持 v1（"SSTB"，u16 长度字段）与 v2（"SSTC"，u32 长度字段）双格式，
+ * 旧库 v1 文件仍可读（小 value 场景无缺陷），新写入使用 v2。
  */
-const SSTABLE_MAGIC = 0x53535442; // "SSTB"
 // ---------------------------------------------------------------------------
 //  SSTableReader
 // ---------------------------------------------------------------------------
@@ -2917,11 +2914,20 @@ class SSTableReader {
         this.indexEntries = [];
         this.entryCount = 0;
         this.bloomFilter = null;
+        /** 格式版本：1 = u16 长度字段（旧），2 = u32 长度字段（v0.4.4） */
+        this.format = 2;
         this.data = data;
         this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
         this.meta = meta;
         this.parseFooter();
     }
+    /** 长度字段宽度：v2 = 4 字节 u32，v1 = 2 字节 u16 */
+    lenFieldSize() {
+        return this.format === 2 ? 4 : 2;
+    }
+    readLen(offset) {
+        return this.format === 2 ? this.view.getUint32(offset, false) : this.view.getUint16(offset, false);
+    }
     // -----------------------------------------------------------------------
     //  查询
     // -----------------------------------------------------------------------
@@ -2935,24 +2941,25 @@ class SSTableReader {
             return null;
         const entry = this.indexEntries[blockIdx];
         const blockData = this.getBlockData(entry);
-        // v0.4.2-fix: 残缺文件（meta 偏移超出实际长度）跳过该块，而非抛 RangeError
+        // v0.4.1-fix: 残缺文件（meta 偏移超出实际长度）跳过该块，而非抛 RangeError
         if (!blockData)
             return null;
         const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
+        const lenSize = this.lenFieldSize();
         const entryCount = blockView.getUint32(0, false);
         let offset = 4;
         // 顺序扫描 block 内的条目（生产中应二分查找）
         for (let i = 0; i < entryCount; i++) {
-            if (offset + 2 > blockData.byteLength)
+            if (offset + lenSize > blockData.byteLength)
                 break;
-            const keyLen = blockView.getUint16(offset, false);
-            offset += 2;
-            if (offset + keyLen + 2 > blockData.byteLength)
+            const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+            offset += lenSize;
+            if (offset + keyLen + lenSize > blockData.byteLength)
                 break;
             const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
             offset += keyLen;
-            const valLen = blockView.getUint16(offset, false);
-            offset += 2;
+            const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+            offset += lenSize;
             if (offset + valLen > blockData.byteLength)
                 break;
             const valBytes = blockData.slice(offset, offset + valLen);
@@ -2976,26 +2983,27 @@ class SSTableReader {
         const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey));
         if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx)
             return;
+        const lenSize = this.lenFieldSize();
         for (let bi = startBlockIdx; bi <= endBlockIdx && bi >= 0; bi++) {
             const entry = this.indexEntries[bi];
             const blockData = this.getBlockData(entry);
-            // v0.4.2-fix: 残缺块跳过（rangeScan 继续后续块，不抛异常）
+            // v0.4.1-fix: 残缺块跳过（rangeScan 继续后续块，不抛异常）
             if (!blockData)
                 continue;
             const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
             const blockEntryCount = blockView.getUint32(0, false);
             let offset = 4;
             for (let i = 0; i < blockEntryCount; i++) {
-                if (offset + 2 > blockData.byteLength)
+                if (offset + lenSize > blockData.byteLength)
                     break;
-                const keyLen = blockView.getUint16(offset, false);
-                offset += 2;
-                if (offset + keyLen + 2 > blockData.byteLength)
+                const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                offset += lenSize;
+                if (offset + keyLen + lenSize > blockData.byteLength)
                     break;
                 const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
                 offset += keyLen;
-                const valLen = blockView.getUint16(offset, false);
-                offset += 2;
+                const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                offset += lenSize;
                 if (offset + valLen > blockData.byteLength)
                     break;
                 const valBytes = blockData.slice(offset, offset + valLen);
@@ -3014,25 +3022,26 @@ class SSTableReader {
     }
     /** 扫描所有条目 */
     scanAll(callback) {
+        const lenSize = this.lenFieldSize();
         for (const entry of this.indexEntries) {
             const blockData = this.getBlockData(entry);
-            // v0.4.2-fix: 残缺块跳过（scanAll 继续后续块，不抛异常）
+            // v0.4.1-fix: 残缺块跳过（scanAll 继续后续块，不抛异常）
             if (!blockData)
                 continue;
             const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
             const blockEntryCount = blockView.getUint32(0, false);
             let offset = 4;
             for (let i = 0; i < blockEntryCount; i++) {
-                if (offset + 2 > blockData.byteLength)
+                if (offset + lenSize > blockData.byteLength)
                     break;
-                const keyLen = blockView.getUint16(offset, false);
-                offset += 2;
-                if (offset + keyLen + 2 > blockData.byteLength)
+                const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                offset += lenSize;
+                if (offset + keyLen + lenSize > blockData.byteLength)
                     break;
                 const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
                 offset += keyLen;
-                const valLen = blockView.getUint16(offset, false);
-                offset += 2;
+                const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
+                offset += lenSize;
                 if (offset + valLen > blockData.byteLength)
                     break;
                 const valBytes = blockData.slice(offset, offset + valLen);
@@ -3063,10 +3072,16 @@ class SSTableReader {
             throw new Error('SSTable too small: missing footer');
         }
         const footerOffset = this.data.byteLength - 32;
-        // 验证魔数
+        // 验证魔数（v1 "SSTB" / v2 "SSTC"）
         const magic = this.view.getUint32(footerOffset + 24, false);
-        if (magic !== SSTABLE_MAGIC) {
-            throw new Error(`Invalid SSTable magic: expected ${SSTABLE_MAGIC}, got ${magic}`);
+        if (magic === SSTABLE_MAGIC_V1) {
+            this.format = 1;
+        }
+        else if (magic === SSTABLE_MAGIC_V2) {
+            this.format = 2;
+        }
+        else {
+            throw new Error(`Invalid SSTable magic: expected ${SSTABLE_MAGIC_V1} or ${SSTABLE_MAGIC_V2}, got ${magic}`);
         }
         const indexOffset = this.view.getUint32(footerOffset, false);
         const indexSize = this.view.getUint32(footerOffset + 4, false);
@@ -3074,7 +3089,7 @@ class SSTableReader {
         const bloomSize = this.view.getUint32(footerOffset + 12, false);
         const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
         this.entryCount = this.view.getUint32(footerOffset + 20, false);
-        // v0.4.2-fix: 完整性校验 — 索引块必须完全落在文件内，否则视为残缺文件跳过
+        // v0.4.1-fix: 完整性校验 — 索引块必须完全落在文件内，否则视为残缺文件跳过
         if (indexOffset + 4 > this.data.byteLength || indexOffset + indexSize > this.data.byteLength) {
             return; // 残缺文件：无索引块可读，get/rangeScan 均返回空
         }
@@ -3094,13 +3109,14 @@ class SSTableReader {
     parseIndexBlock(offset, _size) {
         const entryCount = this.view.getUint32(offset, false);
         offset += 4;
+        const lenSize = this.lenFieldSize();
         for (let i = 0; i < entryCount; i++) {
-            // v0.4.2-fix: 索引条目越界（keyLen/blockOffset/blockSize 超过文件长度）时中止解析，
+            // v0.4.1-fix: 索引条目越界（keyLen/blockOffset/blockSize 超过文件长度）时中止解析，
             // 已解析的有效条目仍可用于查询
-            if (offset + 2 > this.data.byteLength)
+            if (offset + lenSize > this.data.byteLength)
                 break;
-            const keyLen = this.view.getUint16(offset, false);
-            offset += 2;
+            const keyLen = this.readLen(offset);
+            offset += lenSize;
             if (offset + keyLen + 8 > this.data.byteLength)
                 break;
             const key = new TextDecoder().decode(this.data.slice(offset, offset + keyLen));
@@ -3116,7 +3132,7 @@ class SSTableReader {
         }
     }
     /**
-     * v0.4.2-fix: 获取索引条目对应的数据块。
+     * v0.4.1-fix: 获取索引条目对应的数据块。
      * 块偏移/大小越界（残缺 SSTable）时返回 null，由调用方跳过而非抛 RangeError。
      */
     getBlockData(entry) {