From d83910832e5259c010f30c46182a1f529733f606 Mon Sep 17 00:00:00 2001 From: thzxx Date: Mon, 10 Aug 2026 20:14:57 +0800 Subject: [PATCH] =?UTF-8?q?perf(P1):=20=E6=89=B9=E9=87=8F=E6=8F=92?= =?UTF-8?q?=E5=85=A5=E6=80=A7=E8=83=BD=E6=82=AC=E5=B4=96=20=E2=80=94=20ins?= =?UTF-8?q?ert=20=E5=BE=AA=E7=8E=AF=E5=86=85=E9=80=90=E8=A1=8C=20prefetchK?= =?UTF-8?q?eys=EF=BC=88=E6=AF=8F=E8=A1=8C=20await=20drainChain=20=E6=8E=92?= =?UTF-8?q?=E7=A9=BA=E5=90=8E=E5=8F=B0=E9=93=BE=EF=BC=89=EF=BC=8Ccompactio?= =?UTF-8?q?n=20=E5=9C=A8=E9=93=BE=E4=B8=8A=E6=95=B0=E7=A7=92=E6=97=B6?= =?UTF-8?q?=E6=AF=8F=E8=A1=8C=E9=98=BB=E5=A1=9E=E6=95=B0=E7=A7=92=20?= =?UTF-8?q?=E2=86=92=20kv=20=E5=90=8E=E7=AB=AF=2010=20=E4=B8=87=E8=A1=8C?= =?UTF-8?q?=E6=8F=92=E5=85=A5=20353s=EF=BC=9B=E6=94=B9=E4=B8=BA=E6=89=B9?= =?UTF-8?q?=E7=BA=A7=E9=A2=84=E5=8A=A0=E8=BD=BD=E6=9C=AC=E6=89=B9=20PK=20?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=EF=BC=8C=E5=AE=9E=E6=B5=8B=20353s=E2=86=9212?= =?UTF-8?q?.5s=EF=BC=8828=20=E5=80=8D=EF=BC=89/=20opfs=2025s=EF=BC=9B?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=20kv/opfs=20=E5=8F=8C=E5=90=8E=E7=AB=AF=2010?= =?UTF-8?q?=20=E4=B8=87=E8=A1=8C=E5=9B=9E=E5=BD=92=EF=BC=88=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E6=8A=A4=E6=A0=8F+=E7=B4=A2=E5=BC=95=E5=AE=8C?= =?UTF-8?q?=E6=95=B4+=E5=B4=A9=E6=BA=83=E6=81=A2=E5=A4=8D=EF=BC=89?= =?UTF-8?q?=EF=BC=8Caria-cache=20=E6=B5=8B=E8=AF=95=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E9=80=82=E9=85=8D=E6=98=BE=E5=BC=8F=20flush?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 7 +++ src/engine/aria/index.ts | 8 ++- tests/engine/aria-cache.test.ts | 4 ++ tests/engine/aria-prod-load.test.ts | 79 +++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5276465..8f4f211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ All notable changes to MetonaSqlark will be documented in this file. （返回空而非报错，与其余方法不一致）。统一 ensureOpen - **测试盲区** — 快照损坏测试此前篡改未生效（close 后介质不可读），从未真正覆盖损坏路径； 修复并新增 metaSeq 超前回归测试 +- **批量插入性能悬崖（P1）** — insert 循环内逐行 `lsm.prefetchKeys([key])`， + 每行 `await drainChain()` 排空后台链：后台 compaction 在链上数秒时每行阻塞数秒 → + 10 万级插入从 ~5ms/批暴跌到 8~11s/批（kv 后端 10 万行共 353s）。 + 修复：批级预加载本批全部 PK（一次 drainChain），循环内 `lsm.get` 由 + memtable/flush 产物（自动入缓存）兜底，完整性与批量语义不变。 + 实测：kv 后端 10 万行 **353s → 12.5s**（28 倍），opfs 后端 25s； + 新增 kv/opfs 双后端 10 万行回归（性能护栏 + 索引完整 + 崩溃恢复） ### Added diff --git a/src/engine/aria/index.ts b/src/engine/aria/index.ts index ed0b074..b232b70 100644 --- a/src/engine/aria/index.ts +++ b/src/engine/aria/index.ts @@ -537,13 +537,19 @@ export class AriaEngine implements IStorageEngine { // v0.3.1: 批量 WAL 写入（组提交），一次 insert 合并为一次落盘 const walRecords: Omit[] = []; + // v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable（一次 drainChain）。 + // 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链， + // 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖 + // （10 万行 kv 后端从 5ms/批暴跌到 8~11s/批）。批内新数据在 memtable + // 或 flush 产物（自动入缓存），循环内 lsm.get 始终完整。 + await this.lsm.prefetchKeys(rows.map((r) => `${tableName}:${String(r[pkCol])}`)); + for (const row of rows) { const validated = this.validateRow(schema, row); const pkValue = String(validated[pkCol]); const key = `${tableName}:${pkValue}`; // Check duplicate in LSM + transaction snapshot - await this.lsm.prefetchKeys([key]); const existing = this.currentTxnId ? (this.txnSnapshot?.get(key) ?? this.lsm.get(key)) : this.lsm.get(key); diff --git a/tests/engine/aria-cache.test.ts b/tests/engine/aria-cache.test.ts index 4f8d018..00f8481 100644 --- a/tests/engine/aria-cache.test.ts +++ b/tests/engine/aria-cache.test.ts @@ -45,10 +45,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => { await engine.insert('users', makeRows(300)); const lsm = (engine as any).lsm as { + flush(): Promise; getCacheSize(): number; getCacheLimit(): number; getStats(): { sstableCount: number }; }; + // v0.6.1-perf: insert 不再隐式排空后台链（逐行 prefetchKeys 已移除）， + // 显式等待后台 flush 完成后再断言 SSTable 产物 + await lsm.flush(); const stats = lsm.getStats(); // 300 行 / 2KB 阈值 → 应产生多个 SSTable expect(stats.sstableCount).toBeGreaterThan(1); diff --git a/tests/engine/aria-prod-load.test.ts b/tests/engine/aria-prod-load.test.ts index a556c8c..5ff176d 100644 --- a/tests/engine/aria-prod-load.test.ts +++ b/tests/engine/aria-prod-load.test.ts @@ -290,4 +290,83 @@ describe('AriaEngine — 生产负载验证', () => { expect(await engine2.find('big', { table: 'big', where: { tag: 't3' } })).toHaveLength(5000); await engine2.close(); }, 180000); + + it('10 万行 kv 后端（含索引）：完整查询 + 崩溃恢复（v0.6.1-perf 回归）', async () => { + const dbName = uniqueDB(); + const engine = new AriaEngine({ + storageBackend: 'kv', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 30000, + walSyncMode: 'full', + }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + const TOTAL = 100000; + const t0 = Date.now(); + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + const insertMs = Date.now() - t0; + // 性能护栏：修复前 353s（batch 32 起每批 8~11s 性能悬崖），修复后 <30s + console.log(`10万行 kv 插入耗时: ${insertMs}ms`); + expect(insertMs).toBeLessThan(60000); + expect(await engine.count('big')).toBe(TOTAL); + + // 全部 10 个 tag 索引查询完整 + for (let t = 0; t < 10; t++) { + const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } }); + expect(viaIdx.length).toBe(10000); + } + + // 崩溃恢复 + await (engine as any).backend.close(); + (engine as any).opened = false; + const engine2 = new AriaEngine({ + storageBackend: 'kv', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 30000, + walSyncMode: 'full', + }); + await engine2.open(dbName, 1); + expect(await engine2.count('big')).toBe(TOTAL); + expect(await engine2.find('big', { table: 'big', where: { tag: 't7' } })).toHaveLength(10000); + await engine2.close(); + }, 180000); + + it('10 万行 opfs 后端（含索引）：完整查询（v0.6.1-perf 回归）', async () => { + const engine = new AriaEngine({ + storageBackend: 'opfs', + memtableSizeThreshold: 512 * 1024, + checkpointInterval: 30000, + walSyncMode: 'full', + }); + await engine.open(uniqueDB(), 1); + await engine.createTable(SCHEMA()); + + const TOTAL = 100000; + const t0 = Date.now(); + for (let batch = 0; batch < TOTAL / 1000; batch++) { + const rows = [] as Record[]; + for (let i = 0; i < 1000; i++) { + const idx = batch * 1000 + i; + rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` }); + } + await engine.insert('big', rows); + } + const insertMs = Date.now() - t0; + console.log(`10万行 opfs 插入耗时: ${insertMs}ms`); + expect(insertMs).toBeLessThan(90000); + expect(await engine.count('big')).toBe(TOTAL); + for (let t = 0; t < 10; t++) { + const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } }); + expect(viaIdx.length).toBe(10000); + } + await engine.close(); + }, 180000); });