perf(P1): 批量插入性能悬崖 — insert 循环内逐行 prefetchKeys(每行 await drainChain 排空后台链),compaction 在链上数秒时每行阻塞数秒 → kv 后端 10 万行插入 353s;改为批级预加载本批 PK 一次,实测 353s→12.5s(28 倍)/ opfs 25s;新增 kv/opfs 双后端 10 万行回归(性能护栏+索引完整+崩溃恢复),aria-cache 测试同步适配显式 flush
CI / test (18.x) (push) Successful in 18m51s
CI / test (20.x) (push) Successful in 17m42s
CI / test (22.x) (push) Successful in 16m58s
CI / test (24.x) (push) Failing after 14m38s
CI / e2e (push) Successful in 10m36s

This commit is contained in:
thzxx
2026-08-10 20:14:57 +08:00
parent 34225a86b3
commit d83910832e
4 changed files with 97 additions and 1 deletions
+7
View File
@@ -39,6 +39,13 @@ All notable changes to MetonaSqlark will be documented in this file.
(返回空而非报错,与其余方法不一致)。统一 ensureOpen
- **测试盲区** — 快照损坏测试此前篡改未生效(close 后介质不可读),从未真正覆盖损坏路径;
修复并新增 metaSeq 超前回归测试
- **批量插入性能悬崖(P1)** — insert 循环内逐行 `lsm.prefetchKeys([key])`
每行 `await drainChain()` 排空后台链:后台 compaction 在链上数秒时每行阻塞数秒 →
10 万级插入从 ~5ms/批暴跌到 8~11s/批(kv 后端 10 万行共 353s)。
修复:批级预加载本批全部 PK(一次 drainChain),循环内 `lsm.get`
memtable/flush 产物(自动入缓存)兜底,完整性与批量语义不变。
实测:kv 后端 10 万行 **353s → 12.5s**28 倍),opfs 后端 25s
新增 kv/opfs 双后端 10 万行回归(性能护栏 + 索引完整 + 崩溃恢复)
### Added
+7 -1
View File
@@ -537,13 +537,19 @@ export class AriaEngine implements IStorageEngine {
// v0.3.1: 批量 WAL 写入(组提交),一次 insert 合并为一次落盘
const walRecords: Omit<WALRecord, 'lsn' | 'checksum'>[] = [];
// v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable(一次 drainChain)。
// 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链,
// 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖
// 10 万行 kv 后端从 5ms/批暴跌到 8~11s/批)。批内新数据在 memtable
// 或 flush 产物(自动入缓存),循环内 lsm.get 始终完整。
await this.lsm.prefetchKeys(rows.map((r) => `${tableName}:${String(r[pkCol])}`));
for (const row of rows) {
const validated = this.validateRow(schema, row);
const pkValue = String(validated[pkCol]);
const key = `${tableName}:${pkValue}`;
// Check duplicate in LSM + transaction snapshot
await this.lsm.prefetchKeys([key]);
const existing = this.currentTxnId
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
: this.lsm.get(key);
+4
View File
@@ -45,10 +45,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
await engine.insert('users', makeRows(300));
const lsm = (engine as any).lsm as {
flush(): Promise<void>;
getCacheSize(): number;
getCacheLimit(): number;
getStats(): { sstableCount: number };
};
// v0.6.1-perf: insert 不再隐式排空后台链(逐行 prefetchKeys 已移除),
// 显式等待后台 flush 完成后再断言 SSTable 产物
await lsm.flush();
const stats = lsm.getStats();
// 300 行 / 2KB 阈值 → 应产生多个 SSTable
expect(stats.sstableCount).toBeGreaterThan(1);
+79
View File
@@ -290,4 +290,83 @@ describe('AriaEngine — 生产负载验证', () => {
expect(await engine2.find('big', { table: 'big', where: { tag: 't3' } })).toHaveLength(5000);
await engine2.close();
}, 180000);
it('10 万行 kv 后端(含索引):完整查询 + 崩溃恢复(v0.6.1-perf 回归)', async () => {
const dbName = uniqueDB();
const engine = new AriaEngine({
storageBackend: 'kv',
memtableSizeThreshold: 512 * 1024,
checkpointInterval: 30000,
walSyncMode: 'full',
});
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
const TOTAL = 100000;
const t0 = Date.now();
for (let batch = 0; batch < TOTAL / 1000; batch++) {
const rows = [] as Record<string, unknown>[];
for (let i = 0; i < 1000; i++) {
const idx = batch * 1000 + i;
rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` });
}
await engine.insert('big', rows);
}
const insertMs = Date.now() - t0;
// 性能护栏:修复前 353sbatch 32 起每批 8~11s 性能悬崖),修复后 <30s
console.log(`10万行 kv 插入耗时: ${insertMs}ms`);
expect(insertMs).toBeLessThan(60000);
expect(await engine.count('big')).toBe(TOTAL);
// 全部 10 个 tag 索引查询完整
for (let t = 0; t < 10; t++) {
const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } });
expect(viaIdx.length).toBe(10000);
}
// 崩溃恢复
await (engine as any).backend.close();
(engine as any).opened = false;
const engine2 = new AriaEngine({
storageBackend: 'kv',
memtableSizeThreshold: 512 * 1024,
checkpointInterval: 30000,
walSyncMode: 'full',
});
await engine2.open(dbName, 1);
expect(await engine2.count('big')).toBe(TOTAL);
expect(await engine2.find('big', { table: 'big', where: { tag: 't7' } })).toHaveLength(10000);
await engine2.close();
}, 180000);
it('10 万行 opfs 后端(含索引):完整查询(v0.6.1-perf 回归)', async () => {
const engine = new AriaEngine({
storageBackend: 'opfs',
memtableSizeThreshold: 512 * 1024,
checkpointInterval: 30000,
walSyncMode: 'full',
});
await engine.open(uniqueDB(), 1);
await engine.createTable(SCHEMA());
const TOTAL = 100000;
const t0 = Date.now();
for (let batch = 0; batch < TOTAL / 1000; batch++) {
const rows = [] as Record<string, unknown>[];
for (let i = 0; i < 1000; i++) {
const idx = batch * 1000 + i;
rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` });
}
await engine.insert('big', rows);
}
const insertMs = Date.now() - t0;
console.log(`10万行 opfs 插入耗时: ${insertMs}ms`);
expect(insertMs).toBeLessThan(90000);
expect(await engine.count('big')).toBe(TOTAL);
for (let t = 0; t < 10; t++) {
const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } });
expect(viaIdx.length).toBe(10000);
}
await engine.close();
}, 180000);
});