release: v0.4.4 — SSTable 编码修复（UTF-8 字节估算 + u32 长度字段 + v1/v2 双格式兼容），大内容不再崩溃

2026-08-09 21:34:28 +08:00
parent d269bdfb75
commit cff98b0903
21 changed files with 1166 additions and 852 deletions
@@ -23,8 +23,8 @@ import { createSchema } from '../src/table/schema';
 // ---------------------------------------------------------------------------

 describe('[v0.2.5] P0-1: 版本号统一', () => {
-  test('VERSION 常量为当前版本（0.4.3）', () => {
-    expect(VERSION).toBe('0.4.3');
+  test('VERSION 常量为当前版本（0.4.4）', () => {
+    expect(VERSION).toBe('0.4.4');
  });
 });

@@ -400,7 +400,7 @@ describe('[v0.3.3] P1-9: Savepoint + MVCC 一致性', () => {

 describe('[v0.3.3] 端到端', () => {
  test('全部修复点可共存于 MetonaSqlark API', async () => {
-    expect(VERSION).toBe('0.4.3');
+    expect(VERSION).toBe('0.4.4');
    const db = new MetonaSqlark({ name: `e2e-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, mode: 'memory' });
    await db.init();
    await db.defineTable('users', {
@@ -0,0 +1,228 @@
+/**
+ * v0.4.4 回归测试 — SSTable 编码缺陷修复
+ * 覆盖：
+ * - P0: 大段中文内容（300KB）写入 aria → flush → close → reopen 数据完整（缓冲区低估越界）
+ * - P0: >64KB value（u16 长度截断）
+ * - P0: 大 value 超过块大小（4096）→ 单条独立成块读取正确
+ * - P1: v1 旧格式文件兼容读取（旧库数据不丢）
+ * - P1: 中文 key / 混合条目
+ */
+
+import { AriaEngine } from '../src/engine/aria/index';
+import { createSchema } from '../src/table/schema';
+import { SSTableBuilder } from '../src/engine/aria/index/sstable_builder';
+import { SSTableReader } from '../src/engine/aria/index/sstable';
+import type { SSTableMeta } from '../src/engine/aria/types';
+import 'fake-indexeddb/auto';
+
+let idbCounter = 0;
+function uniqueDB(): string {
+  return `r45-${Date.now()}-${++idbCounter}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+const makeMeta = (data: Uint8Array): SSTableMeta => ({
+  id: 1, level: 0, minKey: '', maxKey: '\uffff',
+  blockCount: 1, totalSize: data.byteLength, bloomData: null,
+});
+
+/** 300KB 中文内容（模拟真实笔记/文档数据） */
+function makeBigChineseContent(size = 300 * 1024): string {
+  const chunk = '这是一段用于测试的中文内容。数据持久化必须正确处理 UTF-8 多字节编码！';
+  return chunk.repeat(Math.ceil(size / chunk.length)).slice(0, size);
+}
+
+// ===================================================================
+//  P0: 大内容编码（builder 字节估算）
+// ===================================================================
+
+describe('P0 — SSTable 编码：UTF-8 字节估算', () => {
+  it('构建含 300KB 中文内容的 SSTable 不越界，可完整读回', () => {
+    const builder = new SSTableBuilder(4096);
+    const content = makeBigChineseContent();
+    builder.add('note:1', { id: '1', title: '长文笔记', body: content });
+    builder.add('note:2', { id: '2', body: '小内容' });
+    const { sstableData } = builder.build();
+    // 修复前：缓冲区低估 → build 抛 RangeError
+    const reader = new SSTableReader(sstableData, makeMeta(sstableData));
+    const note1 = reader.get('note:1');
+    expect(note1).not.toBeNull();
+    expect((note1 as any).body).toBe(content);
+    expect((note1 as any).body.length).toBe(content.length);
+  });
+
+  it('构建含 >64KB value 的 SSTable（u32 长度字段，修复前 u16 截断）', () => {
+    const builder = new SSTableBuilder(4096);
+    const content = makeBigChineseContent(70 * 1024); // 70KB > 64KB
+    builder.add('doc:big', { id: 'big', data: content });
+    builder.add('doc:small', { id: 'small', data: 'x' });
+    const { sstableData } = builder.build();
+    const reader = new SSTableReader(sstableData, makeMeta(sstableData));
+    const big = reader.get('doc:big');
+    expect((big as any).data).toBe(content);
+    expect((reader.get('doc:small') as any).data).toBe('x');
+  });
+
+  it('大 value（>4096 块大小）单条独立成块，scanAll/get 均正确', () => {
+    const builder = new SSTableBuilder(64); // 极小块阈值
+    const big = makeBigChineseContent(8 * 1024);
+    for (let i = 0; i < 5; i++) {
+      builder.add(`k-${i}`, { v: i, body: i === 2 ? big : 'small' });
+    }
+    const { sstableData, indexEntries } = builder.build();
+    // 大条目独立成块（块数 > 1）
+    expect(indexEntries.length).toBeGreaterThan(1);
+    const reader = new SSTableReader(sstableData, makeMeta(sstableData));
+    const collected: string[] = [];
+    reader.scanAll((k) => collected.push(k));
+    expect(collected).toEqual(['k-0', 'k-1', 'k-2', 'k-3', 'k-4']);
+    expect((reader.get('k-2') as any).body).toBe(big);
+  });
+
+  it('中文 key（多字节）与混合条目', () => {
+    const builder = new SSTableBuilder(4096);
+    builder.add('表:笔记一', { 标题: '中文标题', 内容: '内容' });
+    builder.add('表:笔记二', { 标题: '第二篇' });
+    const { sstableData } = builder.build();
+    const reader = new SSTableReader(sstableData, makeMeta(sstableData));
+    expect((reader.get('表:笔记一') as any)['标题']).toBe('中文标题');
+    expect((reader.get('表:笔记二') as any)['标题']).toBe('第二篇');
+  });
+
+  it('v2 魔数写入正确（SSTC）', () => {
+    const builder = new SSTableBuilder(4096);
+    builder.add('a', { v: 1 });
+    const { sstableData } = builder.build();
+    const magic = new DataView(sstableData.buffer, sstableData.byteOffset, sstableData.byteLength)
+      .getUint32(sstableData.byteLength - 8, false);
+    expect(magic).toBe(0x53535443); // "SSTC"
+  });
+});
+
+// ===================================================================
+//  P1: v1 旧格式兼容
+// ===================================================================
+
+describe('P1 — v1 旧格式兼容', () => {
+  /** 手工构造 v1 格式（keyLen/valueLen u16 + magic SSTB） */
+  function buildV1SSTable(entries: [string, Record<string, unknown>][]): Uint8Array {
+    const enc = new TextEncoder();
+    const encoded = entries.map(([k, v]) => ({ key: k, keyBytes: enc.encode(k), valueBytes: enc.encode(JSON.stringify(v)) }));
+    // 块（全部放一个块）
+    const entryCount = entries.length;
+    let blockSize = 4;
+    for (const e of encoded) blockSize += 2 + e.keyBytes.length + 2 + e.valueBytes.length;
+    const indexSize = 4 + 2 + encoded[encoded.length - 1].keyBytes.length + 8;
+    const bloomSize = 0;
+    const total = blockSize + indexSize + 32;
+    const buf = new ArrayBuffer(total);
+    const view = new DataView(buf);
+    let offset = 0;
+    view.setUint32(offset, entryCount, false); offset += 4;
+    for (const e of encoded) {
+      view.setUint16(offset, e.keyBytes.length, false); offset += 2;
+      new Uint8Array(buf).set(e.keyBytes, offset); offset += e.keyBytes.length;
+      view.setUint16(offset, e.valueBytes.length, false); offset += 2;
+      new Uint8Array(buf).set(e.valueBytes, offset); offset += e.valueBytes.length;
+    }
+    const indexOffset = offset;
+    view.setUint32(offset, 1, false); offset += 4;
+    const last = encoded[encoded.length - 1];
+    view.setUint16(offset, last.keyBytes.length, false); offset += 2;
+    new Uint8Array(buf).set(last.keyBytes, offset); offset += last.keyBytes.length;
+    view.setUint32(offset, 0, false); offset += 4;
+    view.setUint32(offset, blockSize, false); offset += 4;
+    const footer = offset;
+    view.setUint32(footer, indexOffset, false);
+    view.setUint32(footer + 4, indexSize, false);
+    view.setUint32(footer + 8, 0, false);      // bloom_offset
+    view.setUint32(footer + 12, 0, false);     // bloom_size
+    view.setUint32(footer + 16, 0, false);     // hash_count
+    view.setUint32(footer + 20, entryCount, false);
+    view.setUint32(footer + 24, 0x53535442, false); // SSTB
+    view.setUint32(footer + 28, 0, false);
+    return new Uint8Array(buf);
+  }
+
+  it('v1 格式文件可读（旧库数据不丢）', () => {
+    const v1 = buildV1SSTable([
+      ['u:1', { name: 'Alice', age: 30 }],
+      ['u:2', { name: 'Bob', age: 25 }],
+    ]);
+    const reader = new SSTableReader(v1, makeMeta(v1));
+    expect((reader.get('u:1') as any).name).toBe('Alice');
+    expect((reader.get('u:2') as any).name).toBe('Bob');
+    const keys: string[] = [];
+    reader.scanAll((k) => keys.push(k));
+    expect(keys).toEqual(['u:1', 'u:2']);
+  });
+});
+
+// ===================================================================
+//  P0: 端到端 — 大内容跨重启（爸爸 Electron 冒烟场景）
+// ===================================================================
+
+describe('P0 — 大内容端到端（aria 模式）', () => {
+  it('300KB 中文写入 → close → reopen 数据完整（修复前打开必崩）', async () => {
+    const dbName = uniqueDB();
+    const engine = new AriaEngine({ storageBackend: 'indexeddb', checkpointInterval: 100000 });
+    await engine.open(dbName, 1);
+    await engine.createTable(createSchema('notes', {
+      id: { type: 'string', primaryKey: true },
+      title: { type: 'string' },
+      body: { type: 'string' },
+    }));
+
+    const content = makeBigChineseContent();
+    await engine.insert('notes', [
+      { id: 'n1', title: '长文', body: content },
+      { id: 'n2', title: '短文', body: 'hello' },
+    ]);
+    // 强制落盘（修复前：flush 编码越界崩溃）
+    await (engine as any).lsm.flush();
+    await engine.close();
+
+    // 修复前：打开解析半写文件 → ARIA_OPEN_ERROR
+    const engine2 = new AriaEngine({ storageBackend: 'indexeddb', checkpointInterval: 100000 });
+    await expect(engine2.open(dbName, 1)).resolves.toBeUndefined();
+    const rows = await engine2.find('notes', { table: 'notes' });
+    expect(rows).toHaveLength(2);
+    const byId = Object.fromEntries(rows.map((r) => [r.id, r]));
+    expect(byId['n1'].body).toBe(content);
+    expect(byId['n1'].body.length).toBe(content.length);
+    expect(byId['n2'].body).toBe('hello');
+    await engine2.close();
+  });
+
+  it('多行 >64KB 内容批量写入重开完整（含索引列）', async () => {
+    const dbName = uniqueDB();
+    const engine = new AriaEngine({
+      storageBackend: 'indexeddb',
+      memtableSizeThreshold: 32 * 1024,
+      checkpointInterval: 100000,
+    });
+    await engine.open(dbName, 1);
+    await engine.createTable(createSchema('docs', {
+      id: { type: 'string', primaryKey: true },
+      tag: { type: 'string', index: true },
+      body: { type: 'string' },
+    }));
+    const contents: string[] = [];
+    for (let i = 0; i < 10; i++) {
+      const c = makeBigChineseContent(70 * 1024 + i * 100);
+      contents.push(c);
+      await engine.insert('docs', [{ id: `d${i}`, tag: i % 2 === 0 ? 'even' : 'odd', body: c }]);
+    }
+    await engine.close();
+
+    const engine2 = new AriaEngine({ storageBackend: 'indexeddb', checkpointInterval: 100000 });
+    await engine2.open(dbName, 1);
+    expect(await engine2.count('docs')).toBe(10);
+    const evens = await engine2.find('docs', { table: 'docs', where: { tag: 'even' } });
+    expect(evens).toHaveLength(5);
+    for (let i = 0; i < 10; i++) {
+      const rows = await engine2.find('docs', { table: 'docs', where: { id: `d${i}` } });
+      expect(rows[0].body).toBe(contents[i]);
+    }
+    await engine2.close();
+  });
+});