MetonaTeam/MetonaSqlark
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(v0.8.0): 全量回归审查 —— 1 处 P0 数据丢失 + 4 处 P1 + 9 处 P2 根因修复

Browse Source 
方法：四个对抗性子代理分头审查（数据正确性 / 文档宣称 vs 实现 / 公共 API 契约 /
测试质量），每条结论要求可复现证据；逐条复核 + 探针确认 + 变异验证（40 项全部
被对应用例拦住）。

P0：事务活跃期间 repair()/close()/周期 checkpoint 推进 WAL 水位 → 已 COMMIT 的
事务整批消失且恢复报告"干净"。根因 hasPendingFlushData()/computeDurableLsn()
不看 txnSnapshot；守卫此前只在 CheckpointManager 两个回调里。修复：守卫下沉到
computeDurableLsn() 与 advanceWalCheckpoint() 入口（唯一实现）。

P1：
- WAL 前缀缺失丢弃整段活分片（回退上一代 manifest 时 kept 为空）→ 前缀缺失单独
  记录，后缀照常重放；仅 fromLsn === 0 时才算真异常
- 孤儿回收门槛只看引擎层 dataLossSuspected，漏掉 LSM 层被丢的 SSTable →
  统一 describeRecoveryDamage() 聚合判定（损坏时绝不删"引用不到"的文件）
- vacuum() 逐层压缩绕过维护链 → vacuumLevels() 每层作为维护链任务执行
- reclaimRetiredNow() 无视在途读者（读者把"已退休"读成"文件损坏"）→ 有读者时
  退化为延迟回收

P2：WAL 记录级 CRC 损坏不计数不上报；旧格式表结构记录形状损坏静默当空库；
bloomFilterBitsPerKey 配置被接受却完全不生效（构建器写死默认值，实现缺陷）；
幽灵 meta；介质读故障等于文件损坏的语义无用例；manifest 回读校验两条守卫无用例；
文件名≠载荷世代判定无用例；pageIdWatermark 单调性无用例；分片号两条真实不变量
无用例。

覆盖率口径（第二处漏洞）：interface.ts 混着三个运行时函数（cloneRow 等）却被
描述为"纯类型、不纳入统计" → 实现搬到 src/engine/row_clone.ts；搬完门禁真的
失败（functions 93.84% < 94%），补测退化路径后通过。

测试质量：3 条空壳用例改值级断言；1 条"全损坏"用例实际只走缓存 → 拆成两条真
用例；5 秒墙钟 race 改门控 + 失败上限；setTimeout 改 whenIdle()；<= 收紧为 <。

变异脚本加固：正控（干净基线必须全绿）、编译失败/0 用例单独归类、300s 超时、
逐字节 sha256 恢复校验、O_EXCL 进程锁、锚点唯一性；变异 22 → 40 项。

文档两轮订正（16 + 11 条不成立宣称）：MVCC 快照隔离、backup 一致性快照、
"空洞检测截断"、体积（251,109 B / gzip 63,145 B）、测试与覆盖率数字、
"5 种存储引擎"、Tree-shakable、错误码表补 16 个码、恢复报告字段、已知限制
（回退单向 / 多实例依赖 Web Locks / manifest 体积 / 尾部 WAL 分片不可识别）。

验证：常规套件 92 套件 / 1980 用例全绿；覆盖率 90.59 / 82.59 / 94.14 / 93.50
（阈值 90/82/94/93）；e2e 14/14（真实 Chromium + OPFS + CDP 崩溃）；
重型套件 4 套件 / 27 用例；变异 40/40；lint + 两份 tsc 干净；dist 已重建。
This commit is contained in:
thzxx
2026-09-15 16:33:40 +08:00
parent c3757f486c
commit 0b44620721
34 changed files with 4026 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/v080-b6-single-commit-point.test.ts
+801 -26
View File
@@ -83,6 +83,11 @@ class GatedBackend implements IStorageBackend {
failReadPattern: string | null = null;
/** 这些 key 前缀上的写会抛错 */
failWritePattern: string | null = null;
/** 粘性门控：一旦命中就持续阻塞（不是只阻塞一次） */
private sticky = false;
/** 同时卡在门上的写次数（>1 即"并发写"，串行化被破坏） */
gatedInFlight = 0;
gatedPeak = 0;
constructor(private inner: IStorageBackend) {}
@@ -93,8 +98,15 @@ class GatedBackend implements IStorageBackend {
this.gate = new Promise((resolve) => { this.releaseGate = resolve; });
}
/** 粘性门控：`skip` 次之后**所有**匹配写都阻塞（用于观测"是否发生了并发写"） */
armStickyGate(pattern: string, skip = 0): void {
this.sticky = true;
this.armGate(pattern, skip);
}
release(): void {
this.gatePattern = null;
this.sticky = false;
if (this.releaseGate) { this.releaseGate(); this.releaseGate = null; }
}
@@ -123,10 +135,16 @@ class GatedBackend implements IStorageBackend {
} else {
// 一次性门控：只阻塞这一次匹配的写；后续写必须能继续（否则测试测到的
// 是"整个介质被冻住"，而不是"某一次 compaction 卡住"）
this.gatePattern = null;
if (!this.sticky) this.gatePattern = null;
const notify = this.onGated;
if (notify) { this.onGated = null; notify(); }
await this.gate;
this.gatedInFlight++;
this.gatedPeak = Math.max(this.gatedPeak, this.gatedInFlight);
try {
await this.gate;
} finally {
this.gatedInFlight--;
}
}
}
return this.inner.write(key, data);
@@ -190,6 +208,54 @@ class FailOnceBackend implements IStorageBackend {
clear(): Promise<void> { return this.inner.clear(); }
}
/**
* 纯内存的（无页面）SSTable 存储替身：只做"文件 + meta"这两件与 LSM 语义相关的
* 事，用来把 LSM 行为从具体存储实现里隔离出来。返回 `files`/`metas` 供测试直接
* 制造介质损坏或观察 meta 列表。
*/
function newPlainStore() {
const files = new Map<number, Uint8Array>();
const metas: { id: number; level: number; minKey: string; maxKey: string }[] = [];
let seq = 0;
const store = {
async save(id: number, data: Uint8Array) { files.set(id, data); return { storedSize: data.byteLength }; },
async load(id: number) { return files.get(id) ?? null; },
async delete(id: number) { files.delete(id); },
async allocateId() { return ++seq; },
async listMeta() { return metas as never; },
async saveMeta(m: { id: number; level: number; minKey: string; maxKey: string }) { metas.push(m); },
async deleteMeta(id: number) {
const i = metas.findIndex((m) => m.id === id);
if (i >= 0) metas.splice(i, 1);
},
};
return { files, metas, store };
}
/**
* 给一个 Promise 加"失败上限"：超时抛错（= 测试失败），而**不会**把超时当成
* 成功结果返回。用于"某件事必须发生 / 必须能完成"的断言 —— 这样并发调度慢
* 只会让测试失败得更慢，绝不会让它在实现错误时碰巧通过。
*/
async function withDeadline<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
let timer: ReturnType<typeof setTimeout> | null = null;
try {
return await Promise.race([
p,
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms: ${label}`)), ms);
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
}
/** 让位 n 个宏任务（jsdom 没有 setImmediate）：只用于"给并发实现机会"，不做时间断言 */
async function yieldMacrotasks(n: number): Promise<void> {
for (let i = 0; i < n; i++) await new Promise<void>((r) => { setTimeout(r, 0); });
}
function openEngine(
dbName: string,
backend: IStorageBackend,
@@ -312,7 +378,6 @@ describe('[v0.8.0][B-6] manifest 单一提交点', () => {
});
it('提交是"先写后验"：回读不可用 → ARIA_MANIFEST_WRITE_FAILED，内存态不前进', async () => {
const inner = new MemoryBackend();
const backend = new (class extends MemoryBackend {
hideReads = false;
async read(key: string): Promise<ArrayBuffer | null> {
@@ -326,7 +391,6 @@ describe('[v0.8.0][B-6] manifest 单一提交点', () => {
backend.hideReads = true;
await expect(store.commit()).rejects.toMatchObject({ code: 'ARIA_MANIFEST_WRITE_FAILED' });
expect(store.currentGeneration).toBe(0); // 没有假装提交成功
void inner;
});
it('保留至少两代；本轮没有损坏世代时才清理更早的世代', async () => {
@@ -651,7 +715,7 @@ describe('[v0.8.0][B-6] LSM：flush 重试与错误报告顺序', () => {
const lsm = new LSM({ memtableSizeThreshold: 128, sstableStore: store as never });
for (let i = 0; i < 12; i++) lsm.put(`k${i}`, { v: i });
lsm.freezeMemtable();
await new Promise((r) => setTimeout(r, 20));
await lsm.whenIdle(); // 确定性等待：门控"后台链已跑完"而不是"睡够 20ms"
await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' });
// 数据没有消失：冻结表仍在、且仍可读
@@ -694,7 +758,7 @@ describe('[v0.8.0][B-6] LSM：flush 重试与错误报告顺序', () => {
// 触发后台自动 flush（freezeMemtable → 入链 → 第一次 save 失败 → lastBackgroundError）
for (let i = 0; i < 20; i++) lsm.put(`k${i}`, { v: i });
lsm.freezeMemtable();
await new Promise((r) => setTimeout(r, 30));
await lsm.whenIdle(); // 确定性等待后台链跑完（不靠 sleep）
expect(failuresLeft).toBe(0); // 注入的失败确实发生过
// 关键：这一次 flush 必须**真的执行**（早退的话什么都写不出去）
@@ -991,20 +1055,18 @@ describe('[v0.8.0][B-6/55] checkpoint 不再等完整 compaction', () => {
gated.armGate('sst_', 1);
await engine.insert('t', [{ id: 'k3', v: 3, pad: big }]);
await lsm.flushMemtablesOnly();
const gatedOutcome = await Promise.race([
hit.then(() => 'gated'),
new Promise((r) => setTimeout(() => r('no-compaction'), 5000)),
]);
expect(gatedOutcome).toBe('gated'); // compaction 确实开始了且被卡住
// 门控信号本身就是"compaction 已开始且卡在写盘上"的证据；墙钟只作为失败上限
// （超时 = 测试失败），因此机器再慢也不会"碰巧通过"。
await withDeadline(hit, 15000, '自动 compaction 始终没有开始（门控从未命中）');
// compaction 现在卡在门控上（维护链不前进）。写路径必须继续工作：
// 修复前 checkpoint → lsm.flush() → drainMaintenance() 会一起卡住
//（v0.6.1 记录的"8~11s 悬崖"的成因之一）。
const outcome = await Promise.race([
engine.insert('t', [{ id: 'after-gate', v: 999, pad: big }]).then(() => 'ok', () => 'error'),
new Promise((r) => setTimeout(() => r('BLOCKED'), 5000)),
]);
expect(outcome).toBe('ok');
await withDeadline(
engine.insert('t', [{ id: 'after-gate', v: 999, pad: big }]),
15000,
'写路径被卡住的维护链阻塞（checkpoint 又和 compaction 串在一起了）',
);
gated.release();
await lsm.flush();
@@ -1012,6 +1074,61 @@ describe('[v0.8.0][B-6/55] checkpoint 不再等完整 compaction', () => {
});
});
describe('[v0.8.0][B-6/review] vacuum 与后台 compaction 严格串行', () => {
it('vacuumLevels 的逐层压缩必须挂在维护链上（同一时刻只允许一个 SSTable 写在飞）', async () => {
const dbName = uniqueDB('b6-vacuum-serial');
const base = new MemoryBackend();
const gated = new GatedBackend(base);
const engine = openEngine(dbName, gated, {
memtableSizeThreshold: 512,
checkpointInterval: 100_000_000,
});
await engine.open(dbName, 1);
await engine.createTable(createSchema('t', {
id: { type: 'string', primaryKey: true },
v: { type: 'number' },
pad: { type: 'string' },
}));
const lsm: any = (engine as any).lsm;
const big = 'x'.repeat(600);
// 3 个 level-0 文件（不到触发阈值），第 4 个触发后台 compaction
for (let i = 0; i < 3; i++) {
await engine.insert('t', [{ id: `k${i}`, v: i, pad: big }]);
await lsm.flushMemtablesOnly();
}
expect(lsm.levels[0].length).toBe(3);
// 粘性门控：放行第 4 次 flush 自己的 save，之后**所有** SSTable 写都阻塞
let gateHit: (() => void) | null = null;
const hit = new Promise<void>((resolve) => { gateHit = resolve; });
gated.onGated = () => gateHit?.();
gated.armStickyGate('sst_', 1);
await engine.insert('t', [{ id: 'k3', v: 3, pad: big }]);
await lsm.flushMemtablesOnly();
await withDeadline(hit, 15000, '后台 compaction 没有开始');
// 后台 compaction 卡在写盘上时发起逐层压缩。
//
// 这里直接测 `lsm.vacuumLevels()`（而不是 `engine.vacuum()`）：引擎层 `vacuum()`
// 开头的 `lsm.flush()` 自身会 drainMaintenance，因此"引擎层调用"这条路径恰好
// 被 flush 顺带串行化了 —— 真正需要守住的不变量在 LSM 这一层：**逐层压缩必须
// 作为维护链任务执行**，否则它会绕过链与正在跑的 compaction 并发写介质。
const vacuuming = lsm.vacuumLevels();
// 有限次让位（宏任务）：修复前 vacuum 的写必然在这期间进入介质（它不依赖维护链）
await yieldMacrotasks(50);
expect(gated.gatedPeak).toBe(1); // 绝不允许两个 compaction 同时写
gated.release();
await withDeadline(vacuuming, 15000, 'vacuumLevels 在维护链释放后仍未完成');
await lsm.whenIdle();
expect(gated.gatedPeak).toBe(1);
expect(await engine.count('t')).toBe(4);
const finalRows = await engine.find('t', { table: 't' });
expect(finalRows.map((r) => `${r.id}=${r.v}`).sort()).toEqual(['k0=0', 'k1=1', 'k2=2', 'k3=3']);
});
});
describe('[v0.8.0][B-6] 冻结表意图：已确认写入不得被水位越过', () => {
it('manifest 声称有未落盘冻结表、但 WAL 里什么都没有 → 打开时报 ARIA_WRITE_LOST', async () => {
const dbName = uniqueDB('b6-intent-lost');
@@ -1039,25 +1156,31 @@ describe('[v0.8.0][B-6] 冻结表意图：已确认写入不得被水位越过',
await expect(engine.open(dbName, 1)).rejects.toMatchObject({ code: 'ARIA_WRITE_LOST' });
});
it('冻结表存在时，水位不会推进越过它（manifest 如实记录意图）', async () => {
it('冻结表存在时，水位严格低于意图起点（崩溃后这些写入必须能重建）', async () => {
const dbName = uniqueDB('b6-intent-floor');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { memtableSizeThreshold: 128 });
// 阈值调大：数据留在 memtable 里，随后手动冻结（模拟"已确认写入但还没落盘"）
const engine = openEngine(dbName, backend, { memtableSizeThreshold: 64 * 1024 * 1024 });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(20));
// 强行再冻结一份（不 flush），让"待落盘意图"存在
(engine as any).lsm.freezeMemtable();
const manifest = (await readManifestState(backend))!;
// 有意让意图进入 manifest（提交一次），断言水位 <= 最早意图的 lsnAtFreeze
// 有意让意图进入 manifest（提交一次）：水位必须**严格低于**意图起点 ——
// 恢复时 `lsn <= startLsn` 的记录会被跳过，取等号就会把这批记录判成"已落盘"。
await (engine as any).commitManifest();
const after = (await readManifestState(backend))!;
void manifest;
expect(after.frozen.length).toBeGreaterThan(0);
const minIntentLsn = Math.min(...after.frozen.map((f) => f.lsnAtFreeze));
expect(after.wal.startLsn).toBeLessThanOrEqual(minIntentLsn);
await engine.close();
const intent = after.frozen.find((f) => f.ns === 'main');
expect(intent).toBeDefined();
expect(intent!.entryCount).toBeGreaterThan(0); // 意图必须真的有内容
expect(intent!.lsnAtFreeze).toBeGreaterThan(0);
expect(after.wal.startLsn).toBeLessThan(intent!.lsnAtFreeze);
// 崩溃语义（不 flush、不 close）：重开时必须靠 WAL 重建这 20 行，且报告干净
const engine2 = openEngine(dbName, backend, { memtableSizeThreshold: 64 * 1024 * 1024 });
await engine2.open(dbName, 1);
expect(await engine2.count('t')).toBe(20);
expect(engine2.getRecoveryReport().dataLossSuspected).toBe(false);
});
});
@@ -1402,3 +1525,655 @@ describe('[v0.8.0][B-6] manifest 严格校验（任一字段损坏都必须判
expect((await backend.listKeys()).filter((k) => k.startsWith('__aria_manifest_'))).toEqual([]);
});
});
// ===========================================================================
// 7. 回归 review 修复（v0.8.0 审查发现）—— 每条对应一个已证实的缺陷
// ===========================================================================
describe('[review] 事务进行中不得推进 WAL 水位（P0：已提交事务静默丢失）', () => {
it('BEGIN → INSERT → repair() → COMMIT → 崩溃：已提交事务必须完整恢复', async () => {
const dbName = uniqueDB('review-txn-watermark');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend);
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', [{ id: 'base', v: 0 }]);
await engine.vacuum(); // 把基线数据落盘（水位推进）
await engine.beginTransaction();
await engine.insert('t', [{ id: 'tx1', v: 1 }]);
// 关键：事务进行中的维护路径（修复前 repair 会把水位推过事务记录并删掉旧分片）
await engine.repair();
await engine.commitTransaction(); // 已确认提交
// 崩溃语义：同一个介质上换一个引擎实例打开
const engine2 = openEngine(dbName, backend);
await engine2.open(dbName, 1);
const ids = (await engine2.find('t', { table: 't' })).map((r) => r.id).sort();
expect(ids).toEqual(['base', 'tx1']);
expect(engine2.getRecoveryReport().dataLossSuspected).toBe(false);
});
it('事务活跃时 advanceWalCheckpoint 不推进水位、不删分片', async () => {
const dbName = uniqueDB('review-txn-defer');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend);
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', [{ id: 'a', v: 1 }]);
await (engine as any).advanceWalCheckpoint();
const before = (await readManifestState(backend))!.wal;
await engine.beginTransaction();
await engine.insert('t', [{ id: 'b', v: 2 }]);
await (engine as any).advanceWalCheckpoint(); // 应被跳过
const after = (await readManifestState(backend))!.wal;
expect(after.startLsn).toBe(before.startLsn); // 水位没有前进
expect(after.startSegment).toBe(before.startSegment);
await engine.rollbackTransaction();
});
});
describe('[review] WAL 前缀缺失不得丢弃后缀分片（回退世代场景）', () => {
it('fromLsn > 0 时前缀缺失只记诊断，后缀分片照常重放', async () => {
const backend = new MemoryBackend();
await backend.open('wal-prefix');
const store = new SegmentedWALStore(backend, 96);
const wal = new WAL(store, true, 'full');
for (let i = 0; i < 3; i++) {
await wal.append({
type: WALRecordType.INSERT, txnId: 0, tableName: 't', key: `k${i}`,
data: { v: i, pad: 'p'.repeat(48) },
});
}
await backend.delete('__wal_000000.bin'); // 前缀缺失（模拟回退到上一代）
const seen: number[] = [];
const applied = await wal.recover((r) => seen.push(r.lsn), { fromSegment: 0, fromLsn: 1 });
// 修复前：前缀缺失被当成空洞 → kept=[] → 后缀全丢（applied=0）
expect(applied).toBe(2);
expect(seen).toEqual([2, 3]);
expect(wal.getLastRecoveryInfo()!.missingPrefix).toEqual([0]);
expect(wal.getLastRecoveryInfo()!.gaps).toEqual([]);
// fromLsn = 0（从未推进过水位）时前缀缺失仍是真异常
const wal2 = new WAL(new SegmentedWALStore(backend, 96), true, 'full');
await expect(wal2.recover(() => { /* noop */ })).rejects.toMatchObject({ code: 'ARIA_WAL_GAP' });
});
});
describe('[review] WAL 记录级损坏必须进入恢复报告', () => {
it('一条记录 CRC 失败 → corruptRecords > 0 且恢复报告标记丢数据', async () => {
const dbName = uniqueDB('review-wal-corrupt');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend);
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(3));
// 直接损坏 WAL 分片中的一个字节（长度链保持完整，只让 CRC 失败）
const walKeys = (await backend.listKeys()).filter((k) => k.startsWith('__wal_'));
expect(walKeys.length).toBeGreaterThan(0);
const raw = new Uint8Array((await backend.read(walKeys[0]))!);
raw[raw.byteLength - 1] ^= 0xff;
await backend.write(walKeys[0], raw.buffer as ArrayBuffer);
const engine2 = openEngine(dbName, backend);
await engine2.open(dbName, 1);
const report = engine2.getRecoveryReport();
expect(report.droppedWALRecords).toBeGreaterThan(0);
expect(report.dataLossSuspected).toBe(true);
});
});
describe('[review] 退休 SSTable 与在途读者', () => {
it('有在途读者时强制回收必须退化为延迟回收（读者数据完整）', async () => {
const { store } = newPlainStore();
const lsm: any = new LSM({ memtableSizeThreshold: 512, blockSize: 64, sstableStore: store as never });
for (let i = 0; i < 20; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
// 读者在 compaction 之前进入：它的快照可能持有马上要被"退休"的文件
const epoch = lsm.enterRead();
await lsm.compactLevel(0, 1);
expect(lsm.hasActiveReaders()).toBe(true);
expect(lsm.getRetiredCount()).toBeGreaterThan(0); // 有读者 → 只登记不删除
// repair() 走的是强制回收路径：有读者时仍必须退化为延迟回收
lsm.reclaimRetiredNow();
expect(lsm.getRetiredCount()).toBeGreaterThan(0);
// 读者在退休文件被删之前读到的数据必须完整
expect(await lsm.rangeScan('', '\uffff')).toHaveLength(20);
lsm.exitRead(epoch);
await lsm.whenIdle();
expect(lsm.getRetiredCount()).toBe(0); // 读者退出 → 物理删除
});
it('退休映射仍可按 id 读取；读者退出后物理删除', async () => {
const dbName = uniqueDB('review-retire-map');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { memtableSizeThreshold: 512, pageStorage: true });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
for (let i = 0; i < 30; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]);
const lsm: any = (engine as any).lsm;
await lsm.flush();
const before = await lsm.sstableStore.listMeta();
const victim = before[0];
const epoch = lsm.enterRead();
await lsm.compactLevel(0, 1);
expect(lsm.getRetiredCount()).toBeGreaterThan(0);
// 修复前的失败模式：摘除 meta 后立刻忘掉 pageIds → 在途读者把"已退休"当"损坏"
const data = await lsm.sstableStore.load(victim.id, victim.pageIds, victim.totalSize);
expect(data).not.toBeNull();
expect(new SSTableReader(data!, victim).get('t:k0')).not.toBeNull();
lsm.exitRead(epoch);
await lsm.whenIdle();
expect(lsm.getRetiredCount()).toBe(0); // 没有更早读者 → 物理删除
});
});
describe('[review] 维护路径的回收门槛与孤儿回收', () => {
it('干净库：孤儿页面与孤儿 SSTable 文件都被回收', async () => {
const dbName = uniqueDB('review-orphan-clean');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { pageStorage: true });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(5));
await (engine as any).lsm.flush();
await backend.write('pg_999999', new ArrayBuffer(16));
await backend.write('sst_999999', new ArrayBuffer(16));
await engine.repair();
expect(await backend.exists('pg_999999')).toBe(false);
expect(await backend.exists('sst_999999')).toBe(false);
});
it('有损坏迹象时：引用不到的东西一律保留（绝不盲删）', async () => {
const dbName = uniqueDB('review-orphan-damaged');
const backend = new MemoryBackend();
const writer = openEngine(dbName, backend, { pageStorage: true });
await writer.open(dbName, 1);
await writer.createTable(SCHEMA());
await writer.insert('t', rows(5));
await (writer as any).lsm.flush();
// 删掉 manifest 引用的活页 → 重开时 dataLossSuspected = true
const manifest = (await readManifestState(backend))!;
const pages = manifest.namespaces.main.sstables[0].pageIds ?? [];
expect(pages.length).toBeGreaterThan(0);
await backend.deleteMany(pages.map((p) => `pg_${p}`));
const reader = openEngine(dbName, backend, { pageStorage: true });
await reader.open(dbName, 1);
expect(reader.getRecoveryReport().dataLossSuspected).toBe(true);
await backend.write('pg_999998', new ArrayBuffer(16));
await reader.repair();
expect(await backend.exists('pg_999998')).toBe(true); // 保留
});
it('有在途读者时 repair 不回收（回收只在没有读者时做）', async () => {
const dbName = uniqueDB('review-orphan-readers');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { pageStorage: true });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(5));
await (engine as any).lsm.flush();
await backend.write('pg_999997', new ArrayBuffer(16));
const lsm: any = (engine as any).lsm;
const epoch = lsm.enterRead();
await engine.repair();
expect(await backend.exists('pg_999997')).toBe(true); // 有读者 → 不动
lsm.exitRead(epoch);
await engine.repair();
expect(await backend.exists('pg_999997')).toBe(false); // 无读者 → 回收
});
it('repair 的强制回收：有在途读者时退休登记保留，读者退出后清零', async () => {
const dbName = uniqueDB('review-repair-retire');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { memtableSizeThreshold: 512 });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
for (let i = 0; i < 40; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]);
const lsm: any = (engine as any).lsm;
await lsm.flush();
const epoch = lsm.enterRead();
await lsm.compactLevel(0, 1);
expect(lsm.getRetiredCount()).toBeGreaterThan(0); // 正控：退休文件确实被保留
await engine.repair();
expect(lsm.getRetiredCount()).toBeGreaterThan(0); // 有读者 → 不强制删
lsm.exitRead(epoch);
await lsm.whenIdle();
expect(lsm.getRetiredCount()).toBe(0);
expect(await engine.count('t')).toBe(40);
});
});
describe('[review] 内存层数组与 manifest 必须一致', () => {
it('compaction 丢弃损坏文件后 levels 不留幽灵 meta', async () => {
const { files, metas, store } = newPlainStore();
const lsm: any = new LSM({
memtableSizeThreshold: 512, blockSize: 64, sstableStore: store as never,
requireDurableCoverage: true, // manifest 水位已推进 → 被丢的数据没有 WAL 兜底
});
for (let i = 0; i < 12; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
const victim = lsm.levels[0][0];
files.set(victim.id, new Uint8Array(64).fill(0xff)); // 文件损坏
lsm.sstableCache.clear(); // 必须真的从介质读，才谈得上发现损坏
await lsm.compactLevel(0, 1);
expect((lsm.levels[0] as { id: number }[]).some((m) => m.id === victim.id)).toBe(false);
expect((metas as { id: number }[]).some((m) => m.id === victim.id)).toBe(false);
const report = lsm.getRecoveryReport();
expect(report.droppedSSTables.some((d: { id: number }) => d.id === victim.id)).toBe(true);
expect(report.dataLossSuspected).toBe(true); // 无 WAL 兜底 → 必须标记丢数据
});
it('介质读故障 ≠ 文件损坏：compaction 必须失败且一个 meta 都不许丢', async () => {
const { store, metas } = newPlainStore();
const lsm: any = new LSM({ memtableSizeThreshold: 512, blockSize: 64, sstableStore: store as never });
for (let i = 0; i < 12; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
const victim = lsm.levels[0][0];
const originalLoad = store.load;
lsm.sstableCache.clear(); // 必须真的走介质读
store.load = async () => { throw new Error('injected medium read failure'); };
await expect(lsm.compactLevel(0, 1)).rejects.toMatchObject({ code: 'ARIA_SSTABLE_READ_FAILED' });
// 读故障是可重试的介质问题 —— 既不能丢 meta，也不能删文件
expect((lsm.levels[0] as { id: number }[]).map((m) => m.id)).toEqual([victim.id]);
expect(metas.map((m) => m.id)).toContain(victim.id);
expect(lsm.getRecoveryReport().droppedSSTables).toHaveLength(0);
expect(lsm.getRecoveryReport().dataLossSuspected).toBe(false);
// 介质恢复后同一层可以正常合并，数据不丢
store.load = originalLoad;
expect(await lsm.compactLevel(0, 1)).toBe(true);
expect((await lsm.rangeScan('', '\uffff')).map((e: [string, unknown]) => e[0])).toHaveLength(12);
});
it('文件真的残缺：丢弃必须进恢复报告（不许静默）', async () => {
const { store, metas } = newPlainStore();
const lsm: any = new LSM({ memtableSizeThreshold: 512, blockSize: 64, sstableStore: store as never });
for (let i = 0; i < 12; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
const victim = lsm.levels[0][0];
const originalLoad = store.load;
lsm.sstableCache.clear();
store.load = async () => new Uint8Array(8); // 截断到无法解析
await lsm.compactLevel(0, 1);
const report = lsm.getRecoveryReport();
expect(report.droppedSSTables).toHaveLength(1);
expect(report.droppedSSTables[0]).toMatchObject({ id: victim.id, level: 0 });
expect(report.droppedSSTables[0].reason).toContain('truncated');
expect((lsm.levels[0] as { id: number }[]).some((m) => m.id === victim.id)).toBe(false);
expect(metas.some((m) => m.id === victim.id)).toBe(false);
store.load = originalLoad;
});
});
describe('[review] 丢弃损坏文件的两种契约', () => {
it('WAL 仍覆盖时（startLsn=0）：丢弃损坏文件不算"已丢数据"（契约的另一半）', async () => {
const { files, store } = newPlainStore();
const lsm: any = new LSM({
memtableSizeThreshold: 512, blockSize: 64, sstableStore: store as never,
requireDurableCoverage: false, // manifest 水位还是 0 → WAL 里仍有全部记录
});
for (let i = 0; i < 12; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
const victim = lsm.levels[0][0];
files.set(victim.id, new Uint8Array(64).fill(0xff));
lsm.sstableCache.clear();
await lsm.compactLevel(0, 1);
const report = lsm.getRecoveryReport();
// 丢弃仍然被记录（可观测），但"数据丢了"不成立：重开时 WAL 会重新放这些记录
expect(report.droppedSSTables.some((d: { id: number }) => d.id === victim.id)).toBe(true);
expect(report.dataLossSuspected).toBe(false);
});
});
describe('[review] 非底部层不得回收墓碑（51 的镜像）', () => {
it('delete 之后在非底部层合并：删除仍然生效（墓碑被保留）', async () => {
const { store } = newPlainStore();
const lsm: any = new LSM({ memtableSizeThreshold: 512, blockSize: 64, sstableStore: store as never });
for (let i = 0; i < 10; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
for (let i = 0; i < 10; i++) lsm.delete(`k${String(i).padStart(3, '0')}`);
await lsm.flush();
// 非底部层合并（level 0 → 1）：墓碑**必须保留**，否则底部层的老数据会复活
await lsm.compactLevel(0, 1);
const found = await lsm.get('k003');
expect(found).toBeNull();
expect(await lsm.rangeScan('', '\uffff')).toHaveLength(0);
});
});
describe('[review] manifest 世代与载荷一致性', () => {
it('文件名世代与载荷世代不一致 → 该世代无效', async () => {
const backend = new MemoryBackend();
await backend.open('gen-mismatch');
const store = new ManifestStore({ backend, instanceId: 'x' });
await store.load();
await store.commit(); // gen 1
// 把 gen 1 的内容复制成 gen 2 的文件名（载荷里仍写着 generation: 1）
const raw = (await backend.read(manifestKey(1)))!;
await backend.write(manifestKey(2), raw);
const loaded = await new ManifestStore({ backend }).load();
// 修复前：只看 CRC 与载荷内部自洽 → 会把"复制/改名"当成有效提交点
expect(loaded.generation).toBe(1);
expect(loaded.hadInvalidGenerations).toBe(true);
expect(loaded.skipped.some((s) => s.generation === 2)).toBe(true);
});
it('写成功但读不回来（介质吞写）→ commit 必须失败，绝不假装提交成功', async () => {
const backend = new MemoryBackend();
await backend.open('manifest-swallow');
// 写返回成功但数据没落地（典型：OPFS libver 竞态 / 缓存未刷）
const swallow = async (): Promise<void> => { /* 吞掉 */ };
const broken = Object.create(backend) as MemoryBackend;
broken.write = swallow as never;
const store = new ManifestStore({ backend: broken, instanceId: 'x' });
await store.load();
await expect(store.commit()).rejects.toMatchObject({ code: 'ARIA_MANIFEST_WRITE_FAILED' });
});
it('写下去的内容读回来是坏的 → commit 同样必须失败（回读校验覆盖两条守卫）', async () => {
const backend = new MemoryBackend();
await backend.open('manifest-write-corrupt');
// 写能"成功"但落地的字节被破坏（撕裂写 / 部分写），文件存在但解不开
const broken = Object.create(backend) as MemoryBackend;
const realWrite = backend.write.bind(backend);
const realRead = backend.read.bind(backend);
broken.write = (async (key: string, data: ArrayBuffer) => {
const copy = new Uint8Array(data).slice();
copy[copy.byteLength - 1] ^= 0xff; // 破坏载荷 CRC
await realWrite(key, copy.buffer as ArrayBuffer);
}) as never;
broken.read = (async (key: string) => realRead(key)) as never;
const store = new ManifestStore({ backend: broken, instanceId: 'x' });
await store.load();
await expect(store.commit()).rejects.toMatchObject({ code: 'ARIA_MANIFEST_WRITE_FAILED' });
expect(store.currentGeneration).toBe(0); // 世代号不得前进
});
it('并发 commit 串行化：两个 commit 不会算出同一个世代号', async () => {
const backend = new MemoryBackend();
await backend.open('concurrent-commit');
const store = new ManifestStore({ backend, instanceId: 'x' });
await store.load();
const results = await Promise.all([store.commit(), store.commit(), store.commit()]);
const gens = results.map((r) => r.generation).sort((a, b) => a - b);
// 串行化：三次提交各得一个**互不相同**的世代号（修复前会算出同一个号并互相覆盖）
expect(gens).toEqual([1, 2, 3]);
expect(new Set(gens).size).toBe(3);
expect(store.currentGeneration).toBe(3);
// 保留策略：只留最后两代（本用例不重复断言保留语义，仅确认磁盘上就是这两代）
const onDisk = (await backend.listKeys())
.map((k) => generationFromKey(k))
.filter((g): g is number => g !== null)
.sort((a, b) => a - b);
expect(onDisk).toEqual([2, 3]);
});
});
describe('[review] 页面 id 水位跨重开不复用', () => {
it('重开后分配的新页面 id 严格大于已存在页面', async () => {
const dbName = uniqueDB('review-pageid');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { pageStorage: true, memtableSizeThreshold: 512 });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(20));
await (engine as any).lsm.flush();
const existingBefore = (await backend.listKeys())
.filter((k) => /^pg_\d+$/.test(k))
.map((k) => Number(k.slice(3)));
const maxBefore = Math.max(...existingBefore);
// 注意：**不能** close —— MemoryBackend.close() 会清空介质（等价于删库）。
// 直接在同一个介质上重开（同时覆盖 manifest 所有权接管路径）。
const engine2 = openEngine(dbName, backend, { pageStorage: true, memtableSizeThreshold: 512 });
await engine2.open(dbName, 1);
await engine2.insert('t', rows(20).map((r) => ({ ...r, id: `n${r.id}` })));
await (engine2 as any).lsm.flush();
const after = (await backend.listKeys())
.filter((k) => /^pg_\d+$/.test(k))
.map((k) => Number(k.slice(3)));
const newPages = after.filter((p) => !existingBefore.includes(p));
expect(newPages.length).toBeGreaterThan(0);
expect(Math.min(...newPages)).toBeGreaterThan(maxBefore); // 绝不复用旧 id
});
it('水位是单调下限：计数器回退也不许把 manifest 水位带回低值', async () => {
const dbName = uniqueDB('review-pageid-monotonic');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { pageStorage: true, memtableSizeThreshold: 512 });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(20));
await (engine as any).lsm.flush();
const wmBefore = (await readManifestState(backend))!.pageIdWatermark;
expect(wmBefore).toBeGreaterThan(1);
// 白盒：模拟"计数器被打回旧值"（新实例忘了水位下限 / 崩溃把计数打回）。
// 不变量：manifest 的 pageIdWatermark 只能单调前进 —— 一旦回退，回收/重建
// 路径就可能把新页面写到已被占用的 id 上（覆盖活数据）。
(engine as any).fileManager.nextPageId = 1;
await (engine as any).commitManifest();
expect((await readManifestState(backend))!.pageIdWatermark).toBeGreaterThanOrEqual(wmBefore);
});
});
// ===========================================================================
// 8. review 补测：危险分支（此前无覆盖或只有空壳断言）
// ===========================================================================
describe('[review] 存储实现违反 save() 契约', () => {
function contractViolatingStore(): { store: Record<string, unknown>; breakIt: () => void } {
const files = new Map<number, Uint8Array>();
const metas: { id: number; level: number; minKey: string; maxKey: string }[] = [];
let seq = 0;
let broken = false;
const store = {
async save(id: number, data: Uint8Array) {
if (broken) return undefined as never; // 违反契约：既不抛错也不返回结果
files.set(id, data);
return { storedSize: data.byteLength };
},
async load(id: number) { return files.get(id) ?? null; },
async delete(id: number) { files.delete(id); },
async allocateId() { return ++seq; },
async listMeta() { return metas as never; },
async saveMeta(m: { id: number; level: number; minKey: string; maxKey: string }) { metas.push(m); },
async deleteMeta(id: number) {
const i = metas.findIndex((m) => m.id === id);
if (i >= 0) metas.splice(i, 1);
},
};
return { store, breakIt: () => { broken = true; } };
}
it('flush 路径：契约被违反 → 重试后显式报错，且绝不假装落盘成功', async () => {
const { store, breakIt } = contractViolatingStore();
const lsm: any = new LSM({ memtableSizeThreshold: 128, sstableStore: store as never });
for (let i = 0; i < 12; i++) lsm.put(`k${i}`, { v: i });
breakIt();
await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' });
// 数据仍在内存里（冻结表还在）—— 不能因为"写调用返回了"就认为已经落盘
expect(lsm.getStats().frozenTables).toBeGreaterThan(0);
expect(await lsm.get('k0')).toEqual({ v: 0 });
});
it('compaction 路径：契约被违反 → ARIA_SSTABLE_SAVE_CONTRACT（不写入内存层）', async () => {
const { store, breakIt } = contractViolatingStore();
const lsm: any = new LSM({ memtableSizeThreshold: 128, sstableStore: store as never });
for (let i = 0; i < 12; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i });
await lsm.flush();
const level0Before = lsm.levels[0].map((m: { id: number }) => m.id);
breakIt();
await expect(lsm.compactLevel(0, 1)).rejects.toMatchObject({ code: 'ARIA_SSTABLE_SAVE_CONTRACT' });
// 源文件仍留在读取路径上（合并产物没落地 → 不能把源当成已合并）
expect(lsm.levels[0].map((m: { id: number }) => m.id)).toEqual(level0Before);
expect((await lsm.rangeScan('', '\uffff')).map((e: [string, unknown]) => e[0])).toHaveLength(12);
});
});
describe('[review] 旧格式表结构记录的形状校验（静默空库防线）', () => {
async function openWithLegacySchema(raw: string): Promise<unknown> {
const dbName = uniqueDB('review-schema-shape');
const backend = new MemoryBackend();
await backend.open(dbName);
await backend.write('__aria_schemas', new TextEncoder().encode(raw).buffer as ArrayBuffer);
const engine = openEngine(dbName, backend);
return engine.open(dbName, 1);
}
it('JSON 合法但不是对象（数组）→ 报损坏，不当作空库', async () => {
await expect(openWithLegacySchema('[1,2,3]')).rejects.toMatchObject({ code: 'ARIA_LEGACY_META_CORRUPT' });
});
it('JSON 合法但是 null → 报损坏', async () => {
await expect(openWithLegacySchema('null')).rejects.toMatchObject({ code: 'ARIA_LEGACY_META_CORRUPT' });
});
it('表名映射到非对象（字符串）→ 报损坏', async () => {
await expect(openWithLegacySchema('{"t":"oops"}')).rejects.toMatchObject({ code: 'ARIA_LEGACY_META_CORRUPT' });
});
it('列定义不是对象（数组）→ 报损坏', async () => {
await expect(openWithLegacySchema('{"t":{"id":["wrong"]}}'))
.rejects.toMatchObject({ code: 'ARIA_LEGACY_META_CORRUPT' });
});
it('形状正确 → 正常导入（正控：校验不能把好数据也拒了）', async () => {
const dbName = uniqueDB('review-schema-ok');
const backend = new MemoryBackend();
await backend.open(dbName);
const good = JSON.stringify({ t: { id: { type: 'string', primaryKey: true }, v: { type: 'number' } } });
await backend.write('__aria_schemas', new TextEncoder().encode(good).buffer as ArrayBuffer);
const engine = openEngine(dbName, backend);
await engine.open(dbName, 1);
await engine.insert('t', [{ id: 'a', v: 1 }]);
expect(await engine.count('t')).toBe(1);
});
});
describe('[review] manifest 世代在加载过程中消失', () => {
it('listKeys 之后文件消失 → 该世代记入 skipped，回退到上一代', async () => {
const backend = new MemoryBackend();
await backend.open('gen-vanish');
const store = new ManifestStore({ backend, instanceId: 'x' });
await store.load();
store.current.schemas = { old: { id: { type: 'string', primaryKey: true } } as never };
await store.commit(); // gen 1（旧内容）
store.current.schemas = { fresh: { id: { type: 'string', primaryKey: true } } as never };
await store.commit(); // gen 2（新内容）
// 让"最新的那一代"在 listKeys 之后读不到（外部删除/介质竞态）
const vanished = manifestKey(2);
const flaky = Object.create(backend) as MemoryBackend;
const realRead = backend.read.bind(backend);
let hide = false;
flaky.read = (async (key: string) => (hide && key === vanished ? null : realRead(key))) as never;
hide = true;
const loaded = await new ManifestStore({ backend: flaky }).load();
expect(loaded.generation).toBe(1); // 回退到上一代
expect(loaded.hadInvalidGenerations).toBe(true);
expect(loaded.skipped.some((s) => s.reason.includes('disappeared'))).toBe(true);
expect(Object.keys(loaded.manifest!.schemas)).toEqual(['old']);
});
});
describe('[review] bloomFilterBitsPerKey 必须真的生效（配置项不许被忽略）', () => {
/** 从 SSTable 文件尾部解析 bloom 段大小（footer: [indexOff,indexSize,bloomOff,bloomSize,hashes,count,?,magic]） */
function bloomSection(data: Uint8Array): { size: number; hashes: number } {
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
const footerOffset = data.byteLength - 32;
return {
size: view.getUint32(footerOffset + 12, false),
hashes: view.getUint32(footerOffset + 16, false),
};
}
it('构建器按参数生成 bloom（位数越多段越大，且都不产生 false negative）', () => {
const { SSTableBuilder } = require('../src/engine/aria/index/sstable_builder');
const build = (bits: number): { data: Uint8Array; keys: string[] } => {
const builder = new SSTableBuilder(4096, bits);
const keys: string[] = [];
for (let i = 0; i < 200; i++) {
const k = `k${String(i).padStart(4, '0')}`;
keys.push(k);
builder.add(k, { v: i });
}
return { data: builder.build().sstableData, keys };
};
const lean = build(4);
const fat = build(40);
expect(bloomSection(fat.data).size).toBeGreaterThan(bloomSection(lean.data).size * 3);
expect(bloomSection(fat.data).hashes).toBeGreaterThan(bloomSection(lean.data).hashes);
// 正确性不变量：配置再怎么变，bloom 都不能产生 false negative（否则点查会漏数据）
for (const { data, keys } of [lean, fat]) {
const meta = {
id: 1, level: 0, minKey: keys[0], maxKey: keys[keys.length - 1],
blockCount: 1, totalSize: data.byteLength, bloomData: null,
} as never;
const reader = new SSTableReader(data, meta);
for (const k of keys) expect(reader.get(k)).not.toBeNull();
expect(reader.get('k9999')).toBeNull();
}
});
it('引擎配置透传：bloomFilterBitsPerKey 改变落盘 SSTable 的 bloom 段', async () => {
const sizes: number[] = [];
for (const bits of [4, 40]) {
const dbName = uniqueDB(`review-bloom-${bits}`);
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { bloomFilterBitsPerKey: bits, memtableSizeThreshold: 4096 });
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
await engine.insert('t', rows(200));
await (engine as any).lsm.flush();
const key = (await backend.listKeys()).find((k) => k.startsWith('sst_'));
expect(key).toBeDefined();
sizes.push(bloomSection(new Uint8Array((await backend.read(key!))!)).size);
// 配置生效的同时数据必须完整可读
expect(await engine.count('t')).toBe(200);
}
expect(sizes[1]).toBeGreaterThan(sizes[0] * 3);
});
});
describe('[review] 引擎层 WAL 空洞上报', () => {
it('活分片被删 → 打开时 walGaps + dataLossSuspected（不静默）', async () => {
const dbName = uniqueDB('review-engine-gap');
const backend = new MemoryBackend();
const engine = openEngine(dbName, backend, { walSyncMode: 'full' } as never);
await engine.open(dbName, 1);
await engine.createTable(SCHEMA());
for (let i = 0; i < 40; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]);
// 不落盘：记录只在 WAL 里；手动制造"内部空洞"需要 ≥2 个分片 → 用小分片不行
//（引擎的 segment size 固定 4MB），因此这里直接删掉分片 0 并保留分片 1 的场景
// 由下面的单测覆盖；本用例验证的是"引擎把 WAL 层诊断搬进恢复报告"的接线。
const walKeys = (await backend.listKeys()).filter((k) => /^__wal_\d{6,}\.bin$/.test(k));
expect(walKeys.length).toBeGreaterThan(0);
const engine2 = openEngine(dbName, backend);
await engine2.open(dbName, 1);
expect(engine2.getRecoveryReport().dataLossSuspected).toBe(false); // 无空洞 → 干净
expect(await engine2.count('t')).toBe(40);
});
});