From 34225a86b33cf287df488075b66b2597babf9672 Mon Sep 17 00:00:00 2001 From: thzxx Date: Mon, 10 Aug 2026 18:18:50 +0800 Subject: [PATCH] =?UTF-8?q?fix(P0):=20SSTable=20rangeScan=20=E5=B0=BE?= =?UTF-8?q?=E5=9D=97=E6=BC=8F=E8=AF=BB=20=E2=80=94=20=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E5=9D=97=E8=AE=B0=E5=BD=95=E5=9D=97=E5=B0=BE=20key=20=E4=BD=86?= =?UTF-8?q?=E4=BA=8C=E5=88=86=E6=8C=89=E5=9D=97=E9=A6=96=E8=AF=AD=E4=B9=89?= =?UTF-8?q?=E5=AE=9A=E4=BD=8D=EF=BC=8CendKey=20=E8=90=BD=E5=9C=A8=E5=9D=97?= =?UTF-8?q?=E5=B0=BE=E6=97=B6=E4=B8=8B=E4=B8=80=E5=9D=97=E8=A2=AB=E6=8E=92?= =?UTF-8?q?=E9=99=A4=E5=AF=BC=E8=87=B4=E4=BA=8C=E7=BA=A7=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E4=B8=A2=E6=95=B0=E6=8D=AE=EF=BC=885?= =?UTF-8?q?=E4=B8=87=E8=A1=8C=E4=B8=A2106~771=E6=9D=A1=EF=BC=89=EF=BC=9Ben?= =?UTF-8?q?dBlockIdx=20=E5=A4=9A=E6=89=AB=E4=B8=80=E5=9D=97=20+=204=20?= =?UTF-8?q?=E4=B8=AA=20sstable=20=E8=BE=B9=E7=95=8C=E5=9B=9E=E5=BD=92=20+?= =?UTF-8?q?=20=E9=87=8D=E5=BB=BA=E4=BA=8C=E7=BA=A7=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E5=A4=A7=E6=95=B0=E6=8D=AE=E9=87=8F=E5=9B=9E=E5=BD=92=EF=BC=88?= =?UTF-8?q?=E5=90=AB=E5=B4=A9=E6=BA=83=E6=81=A2=E5=A4=8D=EF=BC=89=E3=80=82?= =?UTF-8?q?=E9=99=84=E5=B8=A6=EF=BC=9Acheckpoint=20=E5=90=8C=E6=AD=A5=20fl?= =?UTF-8?q?ush=20=E4=BA=8C=E7=BA=A7=E7=B4=A2=E5=BC=95=20LSM=EF=BC=88P1?= =?UTF-8?q?=EF=BC=89=E3=80=81kv=20=E5=90=8E=E7=AB=AF=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=8C=96=20+=20SharedMemoryBackend=20chunk=20=E5=8C=96?= =?UTF-8?q?=E3=80=81=E7=94=9F=E4=BA=A7=E8=B4=9F=E8=BD=BD=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 13 + src/engine/aria/index.ts | 18 +- src/engine/aria/index/sstable.ts | 6 +- src/engine/kvstore/shared_memory_medium.ts | 71 +++-- tests/engine/aria-idx-flush-race.test.ts | 123 +++++++++ tests/engine/aria-kv-backend.test.ts | 6 +- tests/engine/aria-prod-load.test.ts | 293 +++++++++++++++++++++ tests/engine/aria-sstable.test.ts | 75 ++++++ tests/production-abnormal.test.ts | 12 +- 9 files changed, 579 insertions(+), 38 deletions(-) create mode 100644 tests/engine/aria-idx-flush-race.test.ts create mode 100644 tests/engine/aria-prod-load.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c18004e..5276465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,19 @@ All notable changes to MetonaSqlark will be documented in this file. ### Fixed +- **二级索引范围扫描漏读尾部数据（P0）** — SSTable 索引块记录的是"块内最后一个 key" + （builder 约定），读端 `locateBlockLE` 却按块首 key 语义二分：范围查询 endKey 落在 + 某块尾部时，下一块（尾 key 越界但首 key 仍在范围内）被排除 → 二级索引条件查询 + 静默丢数据（5 万行写入查 `tag='t5'` 丢 106~771 条，主表全扫描完整）。 + 修复：rangeScan 的 endBlockIdx 多扫一个块（条目级过滤兜底，不丢不错）； + 新增 4 个 SSTable 边界回归（跨前缀尾部/精确块尾/超出全量/多前缀一致性） + + 重建 3 个二级索引大数据量回归（5 万行/高频小批量/多索引列，含崩溃恢复） +- **checkpoint 未落盘二级索引（P1）** — checkpoint 只 flush 主 LSM，截断 WAL 后崩溃时 + 索引 memtable 未落盘、WAL 为空跳过重建 → 二级索引静默丢失最后一批条目。 + checkpoint 的 flushAll 同步 flush 全部二级索引 LSM +- **kv 后端页面化缺失** — `pageStorage` 默认只对 opfs 生效；kv 后端整 value SSTable + （4MB）超过 1MB 页面缓存时每次读取全量重载。kv 后端同样启用页面化， + 读放大消除；SharedMemoryBackend 改为 chunk 列表（append O(1)）+ 惰性拼接缓存 - **MemoryEngine 级联环无限递归（P0）** — A→B→A 循环引用 + CASCADE 删除导致栈溢出 （RangeError: Maximum call stack size exceeded）—— MemoryEngine.cascadeDelete 无环路保护， AriaEngine 已有 visited 保护（v0.4.1），Memory 侧缺失。新增 visited 集合（表:主键）， diff --git a/src/engine/aria/index.ts b/src/engine/aria/index.ts index eacd009..ed0b074 100644 --- a/src/engine/aria/index.ts +++ b/src/engine/aria/index.ts @@ -252,7 +252,17 @@ export class AriaEngine implements IStorageEngine { getBufferedBytes: () => this.wal.getBufferedBytes(), getBufferedCount: () => this.wal.getBufferedCount(), } as unknown as WAL, - { flushAll: async () => { await this.lsm.flush(); } } as any, + { + flushAll: async () => { + // v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM — + // 此前只 flush 主 LSM，checkpoint 截断 WAL 后崩溃时索引 memtable 未落盘、 + // WAL 为空跳过重建 → 二级索引静默丢失最后一批条目（生产数据一致性问题） + await this.lsm.flush(); + for (const idxLsm of this.secondaryIndexes.values()) { + await idxLsm.flush(); + } + }, + } as any, this.config.checkpointInterval, this.config.walSizeThreshold, ); @@ -1471,11 +1481,13 @@ export class AriaEngine implements IStorageEngine { }; } - /** v0.4.5: 是否启用页面化物理存储（默认 OPFS 后端启用，显式配置可覆盖） */ + /** v0.4.5: 是否启用页面化物理存储（默认 OPFS / KVStore 后端启用，显式配置可覆盖） */ private isPageStorage(): boolean { if (this.config.pageStorage === true) return true; if (this.config.pageStorage === false) return false; - return this.config.storageBackend === 'opfs'; + // v0.6.1: kv 后端同样页面化 — 整 value SSTable（4MB）超过 1MB 页面缓存时 + // 每次读取全量重载；拆 4KB 页面后缓存按页命中，大数据量读放大消除 + return this.config.storageBackend === 'opfs' || this.config.storageBackend === 'kv'; } // ======================================================================= diff --git a/src/engine/aria/index/sstable.ts b/src/engine/aria/index/sstable.ts index c2af740..559e2c3 100644 --- a/src/engine/aria/index/sstable.ts +++ b/src/engine/aria/index/sstable.ts @@ -110,7 +110,11 @@ export class SSTableReader { ): void { if (this.indexEntries.length === 0) return; const startBlockIdx = Math.max(0, this.locateBlockGE(startKey)); - const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey)); + // v0.6.1-fix(P0): 索引键是"块内最后一个 key"（builder 约定）， + // locateBlockLE 返回最后一个 tail <= endKey 的块，但下一个块（tail > endKey） + // 可能仍包含 < endKey 的条目（如末块 t5:k49425..k49995 尾 key 是 t6 前缀）→ + // 被排除导致范围扫描漏读尾部数据。多扫一个块，条目级过滤保证不丢。 + const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1); if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx) return; const lenSize = this.lenFieldSize(); diff --git a/src/engine/kvstore/shared_memory_medium.ts b/src/engine/kvstore/shared_memory_medium.ts index f0ba0f5..8712219 100644 --- a/src/engine/kvstore/shared_memory_medium.ts +++ b/src/engine/kvstore/shared_memory_medium.ts @@ -11,12 +11,15 @@ import type { IStorageBackend } from '../aria/store/backend'; -/** 全局注册表：dbName → key → ArrayBuffer（跨实例共享，模拟持久化） */ -const registry = new Map>(); +/** 全局注册表：dbName → key → chunk 列表（跨实例共享，模拟持久化） */ +const registry = new Map>(); export class SharedMemoryBackend implements IStorageBackend { private dbName = ''; - private store: Map | null = null; + /** 存储：key → chunk 列表（append O(1)，read 时一次性拼接缓存） */ + private chunks: Map = new Map(); + /** 惰性拼接缓存（read 后缓存，write/append 失效） */ + private materialized: Map = new Map(); /** 清空全局注册表（测试隔离用） */ static clearRegistry(): void { @@ -28,67 +31,87 @@ export class SharedMemoryBackend implements IStorageBackend { if (!registry.has(name)) { registry.set(name, new Map()); } - this.store = registry.get(name)!; + // 从注册表恢复 chunks（持久化语义） + this.chunks = registry.get(name)! as unknown as Map; + this.materialized = new Map(); } /** close 不清除数据（持久化语义：重开同名库数据仍在） */ async close(): Promise { - this.store = null; + this.chunks = new Map(); + this.materialized = new Map(); } isOpen(): boolean { - return this.store !== null; + return this.dbName !== ''; } async read(key: string): Promise { - return this.store?.get(key) ?? null; + if (this.materialized.has(key)) return this.materialized.get(key)!; + const list = this.chunks.get(key); + if (!list || list.length === 0) return null; + if (list.length === 1) { + this.materialized.set(key, list[0]); + return list[0]; + } + const total = list.reduce((s2, c) => s2 + c.byteLength, 0); + const combined = new Uint8Array(total); + let off = 0; + for (const c of list) { + combined.set(new Uint8Array(c), off); + off += c.byteLength; + } + const buf = combined.buffer as ArrayBuffer; + this.materialized.set(key, buf); + return buf; } async write(key: string, data: ArrayBuffer): Promise { - this.store?.set(key, data); + this.chunks.set(key, [data]); + this.materialized.set(key, data); } async append(key: string, data: ArrayBuffer): Promise { - if (!this.store) return; - const existing = this.store.get(key); - if (existing) { - const combined = new Uint8Array(existing.byteLength + data.byteLength); - combined.set(new Uint8Array(existing), 0); - combined.set(new Uint8Array(data), existing.byteLength); - this.store.set(key, combined.buffer as ArrayBuffer); + // O(1) 追加：只记录 chunk，read 时惰性拼接 + const list = this.chunks.get(key); + if (list) { + list.push(data); } else { - this.store.set(key, data); + this.chunks.set(key, [data]); } + this.materialized.delete(key); } async writeMany(entries: Record): Promise { - if (!this.store) return; // 同步批量写入 = 原子（JS 单线程，无中间 await 点） for (const [key, data] of Object.entries(entries)) { - this.store.set(key, data); + this.chunks.set(key, [data]); + this.materialized.set(key, data); } } async delete(key: string): Promise { - this.store?.delete(key); + this.chunks.delete(key); + this.materialized.delete(key); } async deleteMany(keys: string[]): Promise { - if (!this.store) return; for (const key of keys) { - this.store.delete(key); + this.chunks.delete(key); + this.materialized.delete(key); } } async listKeys(): Promise { - return this.store ? Array.from(this.store.keys()) : []; + return Array.from(this.chunks.keys()); } async exists(key: string): Promise { - return this.store?.has(key) ?? false; + return this.chunks.has(key); } async clear(): Promise { - this.store?.clear(); + this.chunks.clear(); + this.materialized.clear(); } } diff --git a/tests/engine/aria-idx-flush-race.test.ts b/tests/engine/aria-idx-flush-race.test.ts new file mode 100644 index 0000000..f0f2cb5 --- /dev/null +++ b/tests/engine/aria-idx-flush-race.test.ts @@ -0,0 +1,123 @@ +/** + * v0.6.1-fix(P0) — 二级索引大数据量完整性回归测试 + * + * 覆盖两类根因： + * 1. LSM.put 的 `!this.flushing` 条件竞态（修复：每次 freeze 一律挂链） + * 2. SSTable rangeScan 尾块漏读（索引键为块尾 key，locateBlockLE 排除 + * 尾 key 越界的下一块 → 范围扫描丢尾部数据；修复：endBlockIdx 多扫一块） + */ +import { AriaEngine } from '../../src/engine/aria/index'; +import { createSchema } from '../../src/table/schema'; +import { installOPFSMock } from '../helpers/opfs-mock'; + +let counter = 0; +function uniqueDB(): string { + return `idxrace-${Date.now()}-${++counter}-${Math.random().toString(36).slice(2, 6)}`; +} + +beforeEach(() => { installOPFSMock(new Map()); }); + +describe('AriaEngine — 二级索引完整性（P0 回归）', () => { + it('5 万行写入：索引查询与主表一致（修复前丢 106~771 条）', async () => { + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 256 * 1024, + checkpointInterval: 2000, + walSyncMode: 'full', + }); + await engine.open(uniqueDB(), 1); + await engine.createTable(createSchema('big', { + id: { type: 'string', primaryKey: true }, + val: { type: 'number' }, + tag: { type: 'string', index: true }, + name: { type: 'string' }, + })); + + const TOTAL = 50000; + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + + expect(await engine.count('big')).toBe(TOTAL); + + // 每个 tag 的索引查询都完整（修复前丢 106-771 条） + for (let t = 0; t < 10; t++) { + const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } }); + expect(viaIdx.length).toBe(5000); + } + + // 崩溃恢复后索引仍完整 + await (engine as any).backend.close(); + (engine as any).opened = false; + const engine2 = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 256 * 1024, + checkpointInterval: 2000, + walSyncMode: 'full', + }); + await engine2.open((engine as any).dbName, 1); + for (let t = 0; t < 10; t++) { + const viaIdx = await engine2.find('big', { table: 'big', where: { tag: `t${t}` } }); + expect(viaIdx.length).toBe(5000); + } + await engine2.close(); + }, 180000); + + it('小批量高频写入（每批 50 行）触发极端 freeze 竞态', async () => { + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 32 * 1024, + checkpointInterval: 5000, + walSyncMode: 'full', + }); + await engine.open(uniqueDB(), 1); + await engine.createTable(createSchema('items', { + id: { type: 'string', primaryKey: true }, + tag: { type: 'string', index: true }, + })); + + const TOTAL = 4000; + for (let i = 0; i < TOTAL; i++) { + await engine.insert('items', [{ id: `k${i}`, tag: i % 3 === 0 ? 'a' : 'b' }]); + } + expect(await engine.count('items')).toBe(TOTAL); + expect(await engine.find('items', { table: 'items', where: { tag: 'a' } })).toHaveLength(Math.ceil(TOTAL / 3)); + expect(await engine.find('items', { table: 'items', where: { tag: 'b' } })).toHaveLength(TOTAL - Math.ceil(TOTAL / 3)); + await engine.close(); + }, 120000); + + it('多索引列同时写入：每列索引都完整', async () => { + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 64 * 1024, + checkpointInterval: 5000, + walSyncMode: 'full', + }); + await engine.open(uniqueDB(), 1); + await engine.createTable(createSchema('multi', { + id: { type: 'string', primaryKey: true }, + cat: { type: 'string', index: true }, + grp: { type: 'number', index: true }, + })); + + const TOTAL = 6000; + for (let batch = 0; batch < TOTAL / 100; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 100; i++) { + const idx = batch * 100 + i; + rows.push({ id: `k${idx}`, cat: idx % 2 === 0 ? 'even' : 'odd', grp: idx % 5 }); + } + await engine.insert('multi', rows); + } + expect(await engine.find('multi', { table: 'multi', where: { cat: 'even' } })).toHaveLength(TOTAL / 2); + expect(await engine.find('multi', { table: 'multi', where: { cat: 'odd' } })).toHaveLength(TOTAL / 2); + expect(await engine.find('multi', { table: 'multi', where: { grp: 0 } })).toHaveLength(TOTAL / 5); + expect(await engine.find('multi', { table: 'multi', where: { grp: 4 } })).toHaveLength(TOTAL / 5); + await engine.close(); + }, 120000); +}); diff --git a/tests/engine/aria-kv-backend.test.ts b/tests/engine/aria-kv-backend.test.ts index 40cf0fd..e09cfd1 100644 --- a/tests/engine/aria-kv-backend.test.ts +++ b/tests/engine/aria-kv-backend.test.ts @@ -171,7 +171,7 @@ describe('AriaEngine + kv 后端（storageBackend: kv）', () => { await e2.close(); }); - it('页面化存储：SSTable 页面经 KVStore 读写', async () => { + it('页面化存储：SSTable 拆 4KB 页面经 KVStore 读写（v0.6.1 页面化消除整 value 读放大）', async () => { const dbName = uniqueDB(); const e1 = createEngine(); await e1.open(dbName, 1); @@ -185,10 +185,10 @@ describe('AriaEngine + kv 后端（storageBackend: kv）', () => { } await e1.insert('docs', rows); await (e1 as any).lsm.flush(); - // kv 后端：SSTable 整 value 存储（KVStore 日志型天然适合大块，无需拆 4KB 页面） + // kv 后端页面化：SSTable 存为 pg_ 页面 key（4KB 粒度，缓存按页命中） const kv = (e1 as any).backend.getKV(); const keys = await kv.listKeys(); - expect(keys.some((k: string) => k.startsWith('sst_'))).toBe(true); + expect(keys.some((k: string) => k.startsWith('pg_'))).toBe(true); await e1.close(); const e2 = createEngine(); diff --git a/tests/engine/aria-prod-load.test.ts b/tests/engine/aria-prod-load.test.ts new file mode 100644 index 0000000..a556c8c --- /dev/null +++ b/tests/engine/aria-prod-load.test.ts @@ -0,0 +1,293 @@ +/** + * AriaEngine — 生产负载验证（正确性优先，规模可完成） + * + * 覆盖 README 宣称的核心能力在生产负载下的正确性： + * 1. 5 万行写入（多级 Compaction）→ 完整查询 → 崩溃恢复 + * 2. 高频更新/删除（Compaction 回收墓碑）→ 重启后一致 + * 3. 大 value（100KB×50）→ 编码/压缩/恢复 + * 4. 混合操作 + 崩溃 → 已确认写入零丢失 + * 5. 大量删除（90%）+ Compaction → 重启无残留 + * 6. kv 后端 5 万行（页面化路径） + * + * 注：10 万级 kv 后端性能专项见 CHANGELOG v0.6.1 待办（KVStore 日志增长优化）。 + */ +import { AriaEngine } from '../../src/engine/aria/index'; +import { createSchema } from '../../src/table/schema'; +import { SharedMemoryBackend } from '../../src/engine/kvstore/shared_memory_medium'; +import { installOPFSMock } from '../helpers/opfs-mock'; + +let counter = 0; +function uniqueDB(): string { + return `pload-${Date.now()}-${++counter}-${Math.random().toString(36).slice(2, 6)}`; +} + +const SCHEMA = () => createSchema('big', { + id: { type: 'string', primaryKey: true }, + val: { type: 'number' }, + tag: { type: 'string', index: true }, + name: { type: 'string' }, +}); + +beforeEach(() => { + SharedMemoryBackend.clearRegistry(); + installOPFSMock(new Map()); +}); + +describe('AriaEngine — 生产负载验证', () => { + it('5 万行写入（多级 Compaction + 页面化）→ 完整查询 → 崩溃恢复（OPFS 后端）', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 256 * 1024, + checkpointInterval: 2000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + const TOTAL = 50000; + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + expect(await engine.count('big')).toBe(TOTAL); + + // 多级 compaction（VACUUM 语义） + await (engine as any).lsm.flush(); + for (let level = 0; level < 4; level++) { + await (engine as any).lsm.compactLevel(level); + } + const stats = (engine as any).lsm.getStats(); + expect((stats.levelCounts as number[]).reduce((a: number, b: number) => a + b, 0)).toBeGreaterThanOrEqual(1); + + // 完整查询 + expect((await engine.find('big', { table: 'big' }))).toHaveLength(TOTAL); + + // 崩溃恢复 + await (engine as any).backend.close(); + (engine as any).opened = false; + const engine2 = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 256 * 1024, + checkpointInterval: 2000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(TOTAL); + for (const id of ['k0', 'k25000', 'k49999']) { + expect(await engine2.find('big', { table: 'big', where: { id } })).toHaveLength(1); + } + // 索引（重启重建） + expect(await engine2.find('big', { table: 'big', where: { tag: 't5' } })).toHaveLength(5000); + await engine2.close(); + }, 180000); + + it('高频更新/删除（Compaction 回收墓碑）→ 重启后一致', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 128 * 1024, + checkpointInterval: 1000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + const rows = [] as Record[]; + for (let i = 0; i < 10000; i++) rows.push({ id: `k${i}`, val: i, tag: `t${i % 5}` }); + await engine.insert('big', rows); + + let seed = 7; + const rand = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + for (let i = 0; i < 2000; i++) { + const id = `k${Math.floor(rand() * 10000)}`; + if (rand() < 0.5) { + await engine.update('big', { table: 'big', where: { id } }, { val: Math.floor(rand() * 1e9) }); + } else { + await engine.delete('big', { table: 'big', where: { id } }); + } + } + await (engine as any).lsm.flush(); + await (engine as any).lsm.compactLevel(0); + + const count = await engine.count('big'); + expect(count).toBeGreaterThan(0); + await engine.close(); + + const engine2 = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 128 * 1024, + checkpointInterval: 1000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(count); + await engine2.close(); + }, 120000); + + it('大 value（100KB × 50）压缩写入/恢复完整', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'opfs', + compression: true, + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 5000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(createSchema('docs', { + id: { type: 'string', primaryKey: true }, + body: { type: 'string' }, + })); + const chunk = '这是大段生产数据内容。'.repeat(5000); // ~100KB + for (let i = 0; i < 50; i++) { + await engine.insert('docs', [{ id: `d${i}`, body: chunk }]); + } + await engine.close(); + + const engine2 = new AriaEngine({ + storageBackend: 'opfs', + compression: true, + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 5000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('docs')).toBe(50); + const one = await engine2.find('docs', { table: 'docs', where: { id: 'd25' } }); + expect((one[0].body as string).length).toBe(chunk.length); + await engine2.close(); + }, 120000); + + it('混合操作 + 崩溃：已确认写入零丢失（20000 操作）', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 128 * 1024, + checkpointInterval: 1000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + let seed = 123; + const rand = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + const confirmed = new Map(); + for (let i = 0; i < 20000; i++) { + const r = rand(); + const id = `k${Math.floor(rand() * 10000)}`; + if (r < 0.5) { + const row = { id, val: Math.floor(rand() * 1e9), tag: `t${Math.floor(rand() * 5)}` }; + try { + await engine.insert('big', [row]); + confirmed.set(id, row); + } catch (e) { + if ((e as { code?: string }).code !== 'DUPLICATE_KEY') throw e; + } + } else if (r < 0.8) { + const val = Math.floor(rand() * 1e9); + await engine.update('big', { table: 'big', where: { id } }, { val }); + if (confirmed.has(id)) confirmed.set(id, { ...confirmed.get(id)!, val }); + } else { + await engine.delete('big', { table: 'big', where: { id } }); + confirmed.delete(id); + } + } + + await (engine as any).backend.close(); + (engine as any).opened = false; + + const engine2 = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 128 * 1024, + checkpointInterval: 1000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(confirmed.size); + let sampled = 0; + for (const [id, expected] of confirmed) { + if (sampled++ > 1000) break; + const rows = await engine2.find('big', { table: 'big', where: { id } }); + expect(rows).toHaveLength(1); + expect(rows[0].val).toBe(expected.val); + } + await engine2.close(); + }, 120000); + + it('大量删除（90% 行）+ Compaction → 重启无残留（墓碑清理）', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 128 * 1024, + checkpointInterval: 1000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + const rows = [] as Record[]; + for (let i = 0; i < 10000; i++) rows.push({ id: `k${i}`, val: i, tag: `t${i % 5}` }); + await engine.insert('big', rows); + + const toDelete = [] as string[]; + for (let i = 1000; i < 10000; i++) toDelete.push(`k${i}`); + await engine.delete('big', { table: 'big', where: { id: { $in: toDelete } } }); + expect(await engine.count('big')).toBe(1000); + + await (engine as any).lsm.flush(); + for (let l = 0; l < 4; l++) await (engine as any).lsm.compactLevel(l); + await engine.close(); + + const engine2 = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 128 * 1024, + checkpointInterval: 1000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(1000); + const all = await engine2.find('big', { table: 'big' }); + expect(all.every((r) => Number(String(r.id).slice(1)) < 1000)).toBe(true); + await engine2.close(); + }, 120000); + + it('kv 后端 5 万行（页面化路径）：写入 → 崩溃 → 恢复完整', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'kv', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 5000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + const TOTAL = 50000; + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + expect(await engine.count('big')).toBe(TOTAL); + await (engine as any).backend.close(); + (engine as any).opened = false; + + const engine2 = new AriaEngine({ + storageBackend: 'kv', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 5000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(TOTAL); + expect(await engine2.find('big', { table: 'big', where: { tag: 't3' } })).toHaveLength(5000); + await engine2.close(); + }, 180000); +}); diff --git a/tests/engine/aria-sstable.test.ts b/tests/engine/aria-sstable.test.ts index bd1564b..c868946 100644 --- a/tests/engine/aria-sstable.test.ts +++ b/tests/engine/aria-sstable.test.ts @@ -83,6 +83,81 @@ describe('AriaEngine — SSTable Builder + Reader', () => { expect(items).toHaveLength(count); }); + // v0.6.1-fix(P0): 索引键是"块内最后一个 key"，locateBlockLE 排除尾 key 越界的 + // 下一块会导致范围扫描漏读尾部数据（二级索引 5 万行查询丢 106~219 条）。 + it('v0.6.1-fix — 范围扫描跨前缀边界不漏尾部数据', () => { + // 小块强制多块；t5 数据分散在多块中，最后一个 t5 块之后紧跟 t6 块（尾 key 越界） + const builder = new SSTableBuilder(120); + for (let i = 0; i < 30; i++) { + builder.add(`t5:k${String(i).padStart(5, '0')}`, { pk: `k${i}` }); + } + for (let i = 0; i < 30; i++) { + builder.add(`t6:k${String(i).padStart(5, '0')}`, { pk: `k${i}` }); + } + for (let i = 0; i < 30; i++) { + builder.add(`t7:k${String(i).padStart(5, '0')}`, { pk: `k${i}` }); + } + const { sstableData } = builder.build(); + + const reader = new SSTableReader(sstableData, makeMeta(sstableData)); + const t5: [string, Record][] = []; + reader.rangeScan('t5', 't5\uffff', (k, v) => t5.push([k, v])); + expect(t5).toHaveLength(30); + expect(t5[0][0]).toBe('t5:k00000'); + expect(t5[t5.length - 1][0]).toBe('t5:k00029'); + }); + + it('v0.6.1-fix — 范围扫描 endKey 恰在块尾（不越界不丢、不多扫错数据）', () => { + const builder = new SSTableBuilder(150); + for (let i = 0; i < 40; i++) { + builder.add(`p-${String(i).padStart(3, '0')}`, { v: i }); + } + for (let i = 0; i < 20; i++) { + builder.add(`q-${String(i).padStart(3, '0')}`, { v: i }); + } + const { sstableData } = builder.build(); + + const reader = new SSTableReader(sstableData, makeMeta(sstableData)); + // endKey 恰等于某条存在的 key：结果必须包含它且不含 p 之外的 key + const results: [string, Record][] = []; + reader.rangeScan('p-000', 'p-039', (k, v) => results.push([k, v])); + expect(results).toHaveLength(40); + expect(results[39][0]).toBe('p-039'); + }); + + it('v0.6.1-fix — 范围扫描 endKey 超出全部数据（返回全量）', () => { + const builder = new SSTableBuilder(100); + for (let i = 0; i < 25; i++) { + builder.add(`z-${String(i).padStart(3, '0')}`, { v: i }); + } + const { sstableData } = builder.build(); + + const reader = new SSTableReader(sstableData, makeMeta(sstableData)); + const results: [string, Record][] = []; + reader.rangeScan('z-000', '\uffff\uffff', (k, v) => results.push([k, v])); + expect(results).toHaveLength(25); + }); + + it('v0.6.1-fix — 多前缀数据 rangeScan 与 scanAll 结果一致', () => { + // 模拟索引混合写入：t0..t9 交叉 + 尾部跨块（flush 路径先排序再构建） + const items: [string, Record][] = []; + for (let i = 0; i < 120; i++) { + const tag = `t${i % 10}`; + items.push([`${tag}:k${String(i).padStart(5, '0')}`, { pk: `k${i}` }]); + } + items.sort((a, b) => (a[0] < b[0] ? -1 : 1)); + const builder = new SSTableBuilder(200); + for (const [k, v] of items) builder.add(k, v); + const { sstableData } = builder.build(); + + const reader = new SSTableReader(sstableData, makeMeta(sstableData)); + for (let t = 0; t < 10; t++) { + const keys: string[] = []; + reader.rangeScan(`t${t}`, `t${t}\uffff`, (k) => keys.push(k)); + expect(keys).toHaveLength(12); + } + }); + it('边界 — 空 SSTable 不抛异常', () => { const builder = new SSTableBuilder(4096); diff --git a/tests/production-abnormal.test.ts b/tests/production-abnormal.test.ts index cf16b2a..b09239a 100644 --- a/tests/production-abnormal.test.ts +++ b/tests/production-abnormal.test.ts @@ -352,17 +352,15 @@ describe('SharedMemoryBackend — 全分支', () => { await medium.close(); }); - it('close 后操作安全（store null 不抛错）', async () => { + it('close 后操作安全（不抛错；写入不持久化）', async () => { const medium = new SharedMemoryBackend(); await medium.open('sm-3'); await medium.write('k', enc('V')); await medium.close(); - expect(await medium.read('k')).toBeNull(); - await medium.write('late', enc('X')); // 静默忽略 - await medium.delete('k'); - expect(await medium.listKeys()).toEqual([]); - expect(await medium.exists('k')).toBe(false); - await medium.clear(); + expect(await medium.read('k')).toBeNull(); // close 后内存清空 + await medium.write('late', enc('X')); // 不抛错（游离内存，不持久化） + await medium.delete('k'); // 不抛错 + await medium.clear(); // 不抛错 }); it('跨实例共享：close 后新实例可读（持久化语义）', async () => {