perf(P1): 批量插入性能悬崖 — insert 循环内逐行 prefetchKeys(每行 await drainChain 排空后台链),compaction 在链上数秒时每行阻塞数秒 → kv 后端 10 万行插入 353s;改为批级预加载本批 PK 一次,实测 353s→12.5s(28 倍)/ opfs 25s;新增 kv/opfs 双后端 10 万行回归(性能护栏+索引完整+崩溃恢复),aria-cache 测试同步适配显式 flush
This commit is contained in:
@@ -39,6 +39,13 @@ All notable changes to MetonaSqlark will be documented in this file.
|
|||||||
(返回空而非报错,与其余方法不一致)。统一 ensureOpen
|
(返回空而非报错,与其余方法不一致)。统一 ensureOpen
|
||||||
- **测试盲区** — 快照损坏测试此前篡改未生效(close 后介质不可读),从未真正覆盖损坏路径;
|
- **测试盲区** — 快照损坏测试此前篡改未生效(close 后介质不可读),从未真正覆盖损坏路径;
|
||||||
修复并新增 metaSeq 超前回归测试
|
修复并新增 metaSeq 超前回归测试
|
||||||
|
- **批量插入性能悬崖(P1)** — insert 循环内逐行 `lsm.prefetchKeys([key])`,
|
||||||
|
每行 `await drainChain()` 排空后台链:后台 compaction 在链上数秒时每行阻塞数秒 →
|
||||||
|
10 万级插入从 ~5ms/批暴跌到 8~11s/批(kv 后端 10 万行共 353s)。
|
||||||
|
修复:批级预加载本批全部 PK(一次 drainChain),循环内 `lsm.get` 由
|
||||||
|
memtable/flush 产物(自动入缓存)兜底,完整性与批量语义不变。
|
||||||
|
实测:kv 后端 10 万行 **353s → 12.5s**(28 倍),opfs 后端 25s;
|
||||||
|
新增 kv/opfs 双后端 10 万行回归(性能护栏 + 索引完整 + 崩溃恢复)
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
|||||||
@@ -537,13 +537,19 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
// v0.3.1: 批量 WAL 写入(组提交),一次 insert 合并为一次落盘
|
// v0.3.1: 批量 WAL 写入(组提交),一次 insert 合并为一次落盘
|
||||||
const walRecords: Omit<WALRecord, 'lsn' | 'checksum'>[] = [];
|
const walRecords: Omit<WALRecord, 'lsn' | 'checksum'>[] = [];
|
||||||
|
|
||||||
|
// v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable(一次 drainChain)。
|
||||||
|
// 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链,
|
||||||
|
// 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖
|
||||||
|
// (10 万行 kv 后端从 5ms/批暴跌到 8~11s/批)。批内新数据在 memtable
|
||||||
|
// 或 flush 产物(自动入缓存),循环内 lsm.get 始终完整。
|
||||||
|
await this.lsm.prefetchKeys(rows.map((r) => `${tableName}:${String(r[pkCol])}`));
|
||||||
|
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
const validated = this.validateRow(schema, row);
|
const validated = this.validateRow(schema, row);
|
||||||
const pkValue = String(validated[pkCol]);
|
const pkValue = String(validated[pkCol]);
|
||||||
const key = `${tableName}:${pkValue}`;
|
const key = `${tableName}:${pkValue}`;
|
||||||
|
|
||||||
// Check duplicate in LSM + transaction snapshot
|
// Check duplicate in LSM + transaction snapshot
|
||||||
await this.lsm.prefetchKeys([key]);
|
|
||||||
const existing = this.currentTxnId
|
const existing = this.currentTxnId
|
||||||
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
|
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
|
||||||
: this.lsm.get(key);
|
: this.lsm.get(key);
|
||||||
|
|||||||
@@ -45,10 +45,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
|
|||||||
|
|
||||||
await engine.insert('users', makeRows(300));
|
await engine.insert('users', makeRows(300));
|
||||||
const lsm = (engine as any).lsm as {
|
const lsm = (engine as any).lsm as {
|
||||||
|
flush(): Promise<void>;
|
||||||
getCacheSize(): number;
|
getCacheSize(): number;
|
||||||
getCacheLimit(): number;
|
getCacheLimit(): number;
|
||||||
getStats(): { sstableCount: number };
|
getStats(): { sstableCount: number };
|
||||||
};
|
};
|
||||||
|
// v0.6.1-perf: insert 不再隐式排空后台链(逐行 prefetchKeys 已移除),
|
||||||
|
// 显式等待后台 flush 完成后再断言 SSTable 产物
|
||||||
|
await lsm.flush();
|
||||||
const stats = lsm.getStats();
|
const stats = lsm.getStats();
|
||||||
// 300 行 / 2KB 阈值 → 应产生多个 SSTable
|
// 300 行 / 2KB 阈值 → 应产生多个 SSTable
|
||||||
expect(stats.sstableCount).toBeGreaterThan(1);
|
expect(stats.sstableCount).toBeGreaterThan(1);
|
||||||
|
|||||||
@@ -290,4 +290,83 @@ describe('AriaEngine — 生产负载验证', () => {
|
|||||||
expect(await engine2.find('big', { table: 'big', where: { tag: 't3' } })).toHaveLength(5000);
|
expect(await engine2.find('big', { table: 'big', where: { tag: 't3' } })).toHaveLength(5000);
|
||||||
await engine2.close();
|
await engine2.close();
|
||||||
}, 180000);
|
}, 180000);
|
||||||
|
|
||||||
|
it('10 万行 kv 后端(含索引):完整查询 + 崩溃恢复(v0.6.1-perf 回归)', async () => {
|
||||||
|
const dbName = uniqueDB();
|
||||||
|
const engine = new AriaEngine({
|
||||||
|
storageBackend: 'kv',
|
||||||
|
memtableSizeThreshold: 512 * 1024,
|
||||||
|
checkpointInterval: 30000,
|
||||||
|
walSyncMode: 'full',
|
||||||
|
});
|
||||||
|
await engine.open(dbName, 1);
|
||||||
|
await engine.createTable(SCHEMA());
|
||||||
|
|
||||||
|
const TOTAL = 100000;
|
||||||
|
const t0 = Date.now();
|
||||||
|
for (let batch = 0; batch < TOTAL / 1000; batch++) {
|
||||||
|
const rows = [] as Record<string, unknown>[];
|
||||||
|
for (let i = 0; i < 1000; i++) {
|
||||||
|
const idx = batch * 1000 + i;
|
||||||
|
rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` });
|
||||||
|
}
|
||||||
|
await engine.insert('big', rows);
|
||||||
|
}
|
||||||
|
const insertMs = Date.now() - t0;
|
||||||
|
// 性能护栏:修复前 353s(batch 32 起每批 8~11s 性能悬崖),修复后 <30s
|
||||||
|
console.log(`10万行 kv 插入耗时: ${insertMs}ms`);
|
||||||
|
expect(insertMs).toBeLessThan(60000);
|
||||||
|
expect(await engine.count('big')).toBe(TOTAL);
|
||||||
|
|
||||||
|
// 全部 10 个 tag 索引查询完整
|
||||||
|
for (let t = 0; t < 10; t++) {
|
||||||
|
const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } });
|
||||||
|
expect(viaIdx.length).toBe(10000);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 崩溃恢复
|
||||||
|
await (engine as any).backend.close();
|
||||||
|
(engine as any).opened = false;
|
||||||
|
const engine2 = new AriaEngine({
|
||||||
|
storageBackend: 'kv',
|
||||||
|
memtableSizeThreshold: 512 * 1024,
|
||||||
|
checkpointInterval: 30000,
|
||||||
|
walSyncMode: 'full',
|
||||||
|
});
|
||||||
|
await engine2.open(dbName, 1);
|
||||||
|
expect(await engine2.count('big')).toBe(TOTAL);
|
||||||
|
expect(await engine2.find('big', { table: 'big', where: { tag: 't7' } })).toHaveLength(10000);
|
||||||
|
await engine2.close();
|
||||||
|
}, 180000);
|
||||||
|
|
||||||
|
it('10 万行 opfs 后端(含索引):完整查询(v0.6.1-perf 回归)', async () => {
|
||||||
|
const engine = new AriaEngine({
|
||||||
|
storageBackend: 'opfs',
|
||||||
|
memtableSizeThreshold: 512 * 1024,
|
||||||
|
checkpointInterval: 30000,
|
||||||
|
walSyncMode: 'full',
|
||||||
|
});
|
||||||
|
await engine.open(uniqueDB(), 1);
|
||||||
|
await engine.createTable(SCHEMA());
|
||||||
|
|
||||||
|
const TOTAL = 100000;
|
||||||
|
const t0 = Date.now();
|
||||||
|
for (let batch = 0; batch < TOTAL / 1000; batch++) {
|
||||||
|
const rows = [] as Record<string, unknown>[];
|
||||||
|
for (let i = 0; i < 1000; i++) {
|
||||||
|
const idx = batch * 1000 + i;
|
||||||
|
rows.push({ id: `k${idx}`, val: idx, tag: `t${idx % 10}`, name: `User${idx}` });
|
||||||
|
}
|
||||||
|
await engine.insert('big', rows);
|
||||||
|
}
|
||||||
|
const insertMs = Date.now() - t0;
|
||||||
|
console.log(`10万行 opfs 插入耗时: ${insertMs}ms`);
|
||||||
|
expect(insertMs).toBeLessThan(90000);
|
||||||
|
expect(await engine.count('big')).toBe(TOTAL);
|
||||||
|
for (let t = 0; t < 10; t++) {
|
||||||
|
const viaIdx = await engine.find('big', { table: 'big', where: { tag: `t${t}` } });
|
||||||
|
expect(viaIdx.length).toBe(10000);
|
||||||
|
}
|
||||||
|
await engine.close();
|
||||||
|
}, 180000);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user