diff --git a/CHANGELOG.md b/CHANGELOG.md index 5276465..8f4f211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ All notable changes to MetonaSqlark will be documented in this file. (返回空而非报错,与其余方法不一致)。统一 ensureOpen - **测试盲区** — 快照损坏测试此前篡改未生效(close 后介质不可读),从未真正覆盖损坏路径; 修复并新增 metaSeq 超前回归测试 +- **批量插入性能悬崖(P1)** — insert 循环内逐行 `lsm.prefetchKeys([key])`, + 每行 `await drainChain()` 排空后台链:后台 compaction 在链上数秒时每行阻塞数秒 → + 10 万级插入从 ~5ms/批暴跌到 8~11s/批(kv 后端 10 万行共 353s)。 + 修复:批级预加载本批全部 PK(一次 drainChain),循环内 `lsm.get` 由 + memtable/flush 产物(自动入缓存)兜底,完整性与批量语义不变。 + 实测:kv 后端 10 万行 **353s → 12.5s**(28 倍),opfs 后端 25s; + 新增 kv/opfs 双后端 10 万行回归(性能护栏 + 索引完整 + 崩溃恢复) ### Added diff --git a/src/engine/aria/index.ts b/src/engine/aria/index.ts index ed0b074..b232b70 100644 --- a/src/engine/aria/index.ts +++ b/src/engine/aria/index.ts @@ -537,13 +537,19 @@ export class AriaEngine implements IStorageEngine { // v0.3.1: 批量 WAL 写入(组提交),一次 insert 合并为一次落盘 const walRecords: Omit[] = []; + // v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable(一次 drainChain)。 + // 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链, + // 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖 + // (10 万行 kv 后端从 5ms/批暴跌到 8~11s/批)。批内新数据在 memtable + // 或 flush 产物(自动入缓存),循环内 lsm.get 始终完整。 + await this.lsm.prefetchKeys(rows.map((r) => `${tableName}:${String(r[pkCol])}`)); + for (const row of rows) { const validated = this.validateRow(schema, row); const pkValue = String(validated[pkCol]); const key = `${tableName}:${pkValue}`; // Check duplicate in LSM + transaction snapshot - await this.lsm.prefetchKeys([key]); const existing = this.currentTxnId ? (this.txnSnapshot?.get(key) ?? this.lsm.get(key)) : this.lsm.get(key); diff --git a/tests/engine/aria-cache.test.ts b/tests/engine/aria-cache.test.ts index 4f8d018..00f8481 100644 --- a/tests/engine/aria-cache.test.ts +++ b/tests/engine/aria-cache.test.ts @@ -45,10 +45,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => { await engine.insert('users', makeRows(300)); const lsm = (engine as any).lsm as { + flush(): Promise; getCacheSize(): number; getCacheLimit(): number; getStats(): { sstableCount: number }; }; + // v0.6.1-perf: insert 不再隐式排空后台链(逐行 prefetchKeys 已移除), + // 显式等待后台 flush 完成后再断言 SSTable 产物 + await lsm.flush(); const stats = lsm.getStats(); // 300 行 / 2KB 阈值 → 应产生多个 SSTable expect(stats.sstableCount).toBeGreaterThan(1); diff --git a/tests/engine/aria-prod-load.test.ts b/tests/engine/aria-prod-load.test.ts index a556c8c..5ff176d 100644 --- a/tests/engine/aria-prod-load.test.ts +++ b/tests/engine/aria-prod-load.test.ts @@ -290,4 +290,83 @@ describe('AriaEngine — 生产负载验证', () => { expect(await engine2.find('big', { table: 'big', where: { tag: 't3' } })).toHaveLength(5000); await engine2.close(); }, 180000); + + it('10 万行 kv 后端(含索引):完整查询 + 崩溃恢复(v0.6.1-perf 回归)', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'kv', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 30000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + const TOTAL = 100000; + const t0 = Date.now(); + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + const insertMs = Date.now() - t0; + // 性能护栏:修复前 353s(batch 32 起每批 8~11s 性能悬崖),修复后 <30s + console.log(`10万行 kv 插入耗时: ${insertMs}ms`); + expect(insertMs).toBeLessThan(60000); + expect(await engine.count('big')).toBe(TOTAL); + + // 全部 10 个 tag 索引查询完整 + for (let t = 0; t < 10; t++) { + const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } }); + expect(viaIdx.length).toBe(10000); + } + + // 崩溃恢复 + await (engine as any).backend.close(); + (engine as any).opened = false; + const engine2 = new AriaEngine({ + storageBackend: 'kv', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 30000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(TOTAL); + expect(await engine2.find('big', { table: 'big', where: { tag: 't7' } })).toHaveLength(10000); + await engine2.close(); + }, 180000); + + it('10 万行 opfs 后端(含索引):完整查询(v0.6.1-perf 回归)', async () => { + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 30000, + walSyncMode: 'full', + }); + await engine.open(uniqueDB(), 1); + await engine.createTable(SCHEMA()); + + const TOTAL = 100000; + const t0 = Date.now(); + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + const insertMs = Date.now() - t0; + console.log(`10万行 opfs 插入耗时: ${insertMs}ms`); + expect(insertMs).toBeLessThan(90000); + expect(await engine.count('big')).toBe(TOTAL); + for (let t = 0; t < 10; t++) { + const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } }); + expect(viaIdx.length).toBe(10000); + } + await engine.close(); + }, 180000); });