diff --git a/scripts/mutation-b6.py b/scripts/mutation-b6.py new file mode 100644 index 0000000..cab88fd --- /dev/null +++ b/scripts/mutation-b6.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""B-6 变异验证:把每个修复回退到修复前的行为,对应用例必须失败。 + +用法:python3 scripts/mutation-b6.py +任何一条"回退后测试仍然通过"都会以非零退出码报出来(说明测试只是陪跑)。 +""" +import io +import os +import re +import shlex +import signal +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LSM = 'src/engine/aria/index/lsm.ts' +MANIFEST = 'src/engine/aria/store/manifest.ts' +MERGE = 'src/engine/aria/index/merge_iterator.ts' +ENGINE = 'src/engine/aria/index.ts' +SEGSTORE = 'src/engine/aria/wal/segmented_store.ts' +CHECKPOINT = 'src/engine/aria/wal/checkpoint.ts' +SSTABLE = 'src/engine/aria/index/sstable.ts' + +B6 = 'tests/v080-b6-single-commit-point.test.ts' +STRESS = 'tests/engine/aria-repair-hardening.test.ts' + +MUTATIONS = [ + dict( + name='47 MergeIterator 立即补充(提前终止多算 1 条)', + file=MERGE, + old=""" // 胜出来源的补充推迟到下一次 next()(消费者只取 N 条 → 底层只产出 N 条) + this.pendingRefill = first.sourceIndex;""", + new=""" // [MUTATION] 立即补充(修复前行为) + this.seedFromSource(first.sourceIndex);""", + test=B6, pattern='流式扫描提前终止不多算', + ), + dict( + name='49 compacting 单 boolean(跨层触发被丢弃)', + file=LSM, + old=""" private scheduleCompact(level: number): void { + if (level > MAX_LSM_LEVELS - 1 || this.compacting.has(level)) return; + this.compacting.add(level);""", + new=""" private scheduleCompact(level: number): void { + if (level > MAX_LSM_LEVELS - 1 || this.compacting.size > 0) return; // [MUTATION] 单 boolean 语义 + this.compacting.add(level);""", + test=B6, pattern='按层 compaction 状态', + ), + dict( + name='50 compaction 先 splice 整层(窗口内该层不可见)', + file=LSM, + old=""" const selected = [...this.levels[level]]; + const selectedIds = new Set(selected.map((m) => m.id));""", + new=""" const selected = this.levels[level].splice(0, this.levels[level].length); // [MUTATION] 旧行为 + const selectedIds = new Set(selected.map((m) => m.id));""", + test=B6, pattern='compaction 期间该层对读者始终可见', + ), + dict( + name='51 底部层不回收墓碑', + file=LSM, + old=""" const merged = isBottomLevel + ? mergedRaw.filter(([, value]) => !(value as unknown as Record).__tombstone) + : mergedRaw;""", + new=""" const merged = mergedRaw; // [MUTATION] 不回收墓碑""", + test=B6, pattern='底部层合并回收墓碑', + ), + dict( + name='44 冻结表失败后不重试', + file=LSM, + old=""" private async retryPendingFlushes(): Promise { + let rounds = 0;""", + new=""" private async retryPendingFlushes(): Promise { + return; // [MUTATION] 无重试路径 + let rounds = 0;""", + test=B6, pattern='持续失败 → flush 明确报错且数据不丢', + ), + dict( + name='45 后台错误检查放回"入链之前"', + file=LSM, + old=""" async flush(): Promise { + this.enqueuePendingMemtables(); + const reported = this.consumeBackgroundError(); + await this.drainChain();""", + new=""" async flush(): Promise { + // [MUTATION] 旧行为:先检查/报告后台错误 → 本次 flush 被整个跳过 + const early = this.consumeBackgroundError(); + if (early !== null) { + throw new DatabaseError('background error (legacy behaviour)', 'ARIA_BACKGROUND_ERROR', early); + } + this.enqueuePendingMemtables(); + const reported = this.consumeBackgroundError(); + await this.drainChain();""", + test=B6, pattern='后台错误不得让本次 flush 白做', + ), + dict( + name='manifest 全世代损坏 → 返回空状态(静默空库)', + file=MANIFEST, + old=""" if (generations.length > 0) { + // 文件存在但一代都读不出来:这是元数据损坏,不是空库 + throw new DatabaseError(""", + new=""" if (false && generations.length > 0) { // [MUTATION] 静默当空库 + throw new DatabaseError(""", + test=B6, pattern='全部世代都损坏', + ), + dict( + name='陈旧实例不设防(静默覆盖新世代)', + file=MANIFEST, + old=""" if (!opts.allowTakeover && newestForeign > this.generation) {""", + new=""" if (false && !opts.allowTakeover && newestForeign > this.generation) { // [MUTATION] 不设防""", + test=B6, pattern='陈旧实例提交被拒绝', + ), + dict( + name='旧格式 meta 损坏 → 返回空(静默空库)', + file=ENGINE, + old=""" } catch (error) { + throw new DatabaseError( + `Legacy SSTable metadata "${key}" is corrupt and cannot be migrated: ${(error as Error).message}`, + 'ARIA_LEGACY_META_CORRUPT', + error, + ); + }""", + new=""" } catch (error) { + continue; // [MUTATION] 静默跳过损坏的旧 meta + }""", + test=B6, pattern='旧格式 meta 损坏', + ), + dict( + name='介质读故障被当成"文件不存在"(误删 meta)', + file=LSM, + old=""" } catch (error) { + throw new DatabaseError( + `AriaEngine failed to read SSTable id=${meta.id} from storage: ${(error as Error).message}`, + 'ARIA_SSTABLE_READ_FAILED', + error, + ); + }""", + new=""" } catch (error) { + await this.dropInvalidSSTable(meta, 'read failure treated as missing'); // [MUTATION] 旧行为 + return null; + }""", + test=B6, pattern='介质读故障 ≠ 文件缺失', + ), + dict( + name='WAL 分片空洞静默丢弃尾部', + file='src/engine/aria/wal/log.ts', + old=""" if (gaps.length > 0 && !opts.allowGaps) {""", + new=""" if (false && gaps.length > 0 && !opts.allowGaps) { // [MUTATION] 静默丢弃""", + test=B6, pattern='活跃区间内缺失分片', + ), + dict( + name='WAL 分片号复用(旧世代排在新记录之后)', + file=SEGSTORE, + old=""" if (leftover.length === 0) { + this.currentSegment = Math.max(maxSeq + 1, keepFrom);""", + new=""" if (leftover.length === 0) { + this.currentSegment = 0; // [MUTATION] 复用分片号""", + test=STRESS, pattern='随机操作压力', + ), + dict( + name='冻结表意图不做"写入是否真的丢了"校验', + file=ENGINE, + old=""" const summary = intents.map((i) => `${i.ns}#${i.id}(${i.entryCount} 项)`).join(', '); + throw new DatabaseError(""", + new=""" const summary = intents.map((i) => `${i.ns}#${i.id}(${i.entryCount} 项)`).join(', '); + if (summary) return; // [MUTATION] 不校验 + throw new DatabaseError(""", + test=B6, pattern='ARIA_WRITE_LOST', + ), + dict( + name='提交中的冻结表仍写进意图(回退时误报 WRITE_LOST)', + file=LSM, + old=""" // 正在提交的表由当前这次 manifest 提交负责(见 FrozenTable.committing) + if (frozen.committing) continue;""", + new=""" // [MUTATION] 提交中的表也写进意图""", + test=B6, pattern='回退上一代并标记 manifestFallback', + ), + dict( + name='checkpoint 等完整 compaction(写路径被拖住)', + file=CHECKPOINT, + old=""" if (this.flushable && typeof this.flushable.flushMemtables === 'function') {""", + new=""" if (false && this.flushable && typeof this.flushable.flushMemtables === 'function') { // [MUTATION] 旧行为""", + test=B6, pattern='checkpoint 不再等完整 compaction', + ), + dict( + name='collectReaders 不做结构版本重试(扫描漏掉并发发布的 SSTable)', + file=LSM, + old=""" if (version === this.structureVersion) return readers; + this.readStructureRetries++; + }""", + new=""" return readers; // [MUTATION] 不校验结构版本 + }""", + test=B6, pattern='并发 flush 与 compaction 交错时', + ), + dict( + name='点查不做结构版本重试(get 的校验失效)', + file=LSM, + old=""" if (version !== this.structureVersion) { + this.readStructureRetries++; + continue; // 期间有结构变化(新产物发布 / 前台冻结)→ 重来 + }""", + new=""" if (false) { + this.readStructureRetries++; + continue; + }""", + test=B6, pattern='并发 flush 与 compaction 交错时', + ), + dict( + name='冻结 memtable 不更新结构版本(点查读到旧值/漏数据)', + file=LSM, + old=""" // v0.8.0:**前台变化也算结构变化**。读路径的"结构版本一致"必须覆盖""", + new=""" this.structureVersion--; // [MUTATION] 抵消下面的自增 + // v0.8.0:**前台变化也算结构变化**。读路径的"结构版本一致"必须覆盖""", + test=B6, pattern='并发 flush 与 compaction 交错时', + ), + dict( + name='sstable 越界策略不统一(scanAll 另有一套实现:越界即整体放弃)', + file=SSTABLE, + old=""" scanAll(callback: (key: string, value: Record) => void): void { + for (const [key, value] of this.iterEntries(0, this.indexEntries.length - 1)) { + callback(key, value); + } + }""", + new=""" scanAll(callback: (key: string, value: Record) => void): void { + // [MUTATION] 修复前的状态:这里另有一份解析循环,越界策略与 iterEntries 不同 + const lenSize = this.lenFieldSize(); + for (let bi = 0; bi < this.indexEntries.length; bi++) { + const blockData = this.getBlockData(this.indexEntries[bi]); + if (!blockData) continue; + const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength); + const count = blockView.getUint32(0, false); + let offset = 4; + for (let i = 0; i < count; i++) { + if (offset + lenSize > blockData.byteLength) return; + const keyLen = blockView.getUint32(offset, false); + offset += lenSize; + if (offset + keyLen + lenSize > blockData.byteLength) return; + const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen)); + offset += keyLen; + const valLen = blockView.getUint32(offset, false); + offset += lenSize; + if (offset + valLen > blockData.byteLength) return; + const valBytes = blockData.slice(offset, offset + valLen); + offset += valLen; + try { callback(key, JSON.parse(new TextDecoder().decode(valBytes))); } catch { /* skip */ } + } + } + }""", + test=B6, pattern='三个读取路径对同一个损坏文件给出一致结论', + ), + dict( + name='vacuum 硬编码返回 6(报告与事实无关)', + file=ENGINE, + old=""" let compactedLevels = 0; + const isBottom = (level: number): boolean => level === MAX_LSM_LEVELS - 1;""", + new=""" let compactedLevels = 6; // [MUTATION] 旧行为:与事实无关的数字 + const isBottom = (level: number): boolean => level === MAX_LSM_LEVELS - 1;""", + test=B6, pattern='vacuum 返回', + ), + dict( + name='close() 落盘失败后不释放后端/锁(无 try/finally)', + file=ENGINE, + old=""" let failure: unknown = null; + try { + await this.flushAllLsms();""", + new=""" let failure: unknown = null; + if (true) { // [MUTATION] 旧行为:失败即中断,收尾逻辑不再执行 + await this.flushAllLsms(); + await this.bufferPool.flushAll(); + await this.advanceWalCheckpoint(); + await this.backend.close(); + if (this.dbLock) { await this.dbLock.release(); this.dbLock = null; } + this.schemas.clear(); + this.opened = false; + return; + } + try { + await this.flushAllLsms();""", + test=B6, pattern='close\\(\\) 在落盘失败时', + ), + dict( + name='恢复报告不聚合 LSM 侧丢弃(数据丢失不可见)', + file=ENGINE, + old=""" for (const lsm of this.allLsms()) { + const r = lsm.getRecoveryReport(); + for (const d of r.droppedSSTables) { + dropped.push({ namespace: r.namespace, ...d }); + } + if (r.dataLossSuspected) dataLoss = true; + }""", + new=""" // [MUTATION] 不聚合 LSM 报告""", + test=B6, pattern='恢复报告聚合', + ), +] + + +CURRENT = {'path': None, 'content': None} + + +def _restore_current(*_args): + """收到信号时也要把源码恢复原状(否则中断会留下变异源码)。""" + if CURRENT['path'] and CURRENT['content'] is not None: + io.open(CURRENT['path'], 'w', encoding='utf-8').write(CURRENT['content']) + print(f"\n[restore] {CURRENT['path']} 已恢复", flush=True) + sys.exit(130) + + +for _sig in ('SIGINT', 'SIGTERM', 'SIGHUP'): + try: + signal.signal(getattr(signal, _sig), _restore_current) + except (AttributeError, ValueError): # pragma: no cover - 平台差异 + pass + + +def run(cmd): + return subprocess.run(cmd, cwd=ROOT, shell=True, capture_output=True, text=True) + + +def main(): + only = sys.argv[1] if len(sys.argv) > 1 else None + results = [] + for m in MUTATIONS: + if only and only not in m['name']: + continue + path = os.path.join(ROOT, m['file']) + original = io.open(path, encoding='utf-8').read() + if m['old'] not in original: + results.append((m['name'], 'SKIP(锚点未找到)')) + print(f"[skip] {m['name']}: 锚点未找到", flush=True) + continue + mutated = original.replace(m['old'], m['new'], 1) + CURRENT['path'] = path + CURRENT['content'] = original + io.open(path, 'w', encoding='utf-8').write(mutated) + try: + # 注意:用 shlex.quote 而不是 repr —— repr 会把正则里的反斜杠再转义一次, + # 于是 jest 收到 `\\(` 这种模式、匹配不到任何用例,脚本会把"没跑用例" + # 误判成"测试仍然通过"。 + cmd = (f"npx jest {shlex.quote(m['test'])} -t {shlex.quote(m['pattern'])} " + f"--testPathIgnorePatterns='/node_modules/' 2>&1 | tail -80") + out = run(cmd).stdout + # 注意:不能只搜 "0 total" —— jest 的 "Snapshots: 0 total" 也会命中, + # 于是把"跑了用例且失败"误判成"用例未匹配"。 + no_tests = bool(re.search(r'^Tests:\s+0 total', out, re.M)) or ('No tests found' in out) + failed = ('✕' in out) or ('Tests:' in out and 'failed' in out) or ('●' in out) + if no_tests: + status = 'SKIP(用例未匹配)' + elif failed: + status = 'FAIL(如预期)' + else: + status = 'PASS(!!)' + results.append((m['name'], status)) + msg = { + 'FAIL(如预期)': '测试失败(变异被拦住)', + 'PASS(!!)': '测试仍然通过!', + 'SKIP(用例未匹配)': '用例未匹配(模式写错了)', + }[status] + print(f"[{'ok' if status == 'FAIL(如预期)' else 'BAD'}] {m['name']} → {msg}", flush=True) + finally: + io.open(path, 'w', encoding='utf-8').write(original) + CURRENT['path'] = None + CURRENT['content'] = None + print('\n==== 变异验证汇总 ====') + bad = [r for r in results if r[1] != 'FAIL(如预期)'] + for name, status in results: + print(f' {status:14s} {name}') + if bad: + print(f'\n{len(bad)} 条变异没有被测试拦住 —— 那些用例只是陪跑。') + return 1 + print('\n全部变异都被对应用例拦住。') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/engine/aria/index.ts b/src/engine/aria/index.ts index 427bc5e..4406f28 100644 --- a/src/engine/aria/index.ts +++ b/src/engine/aria/index.ts @@ -14,7 +14,7 @@ import { stripUndefinedUpdates } from '../../table/schema'; import { compileValidator } from '../../table/validation'; import type { AriaEngineConfig, SSTableMeta } from './types'; -import { DEFAULT_ARIA_CONFIG } from './types'; +import { DEFAULT_ARIA_CONFIG, MAX_LSM_LEVELS } from './types'; import { LSM } from './index/lsm'; import type { SSTableStore } from './index/lsm'; @@ -29,11 +29,31 @@ import { OPFSBackend } from './store/opfs_backend'; import { EncryptedBackend } from './store/encrypted_backend'; import { PageSSTableStore } from './store/page_sstable_store'; import { FileManager } from './store/file_manager'; +import { + ManifestStore, + createEmptyManifest, + type AriaManifest, + type ManifestFrozenIntent, +} from './store/manifest'; import { MVCCManager } from './transaction/mvcc'; import { BufferPool } from './buffer/pool'; import { compressLZ4, decompressLZ4 } from './compression/lz4'; import type { WALRecord as _WALRecord } from './types'; +/** v0.8.0:引擎恢复诊断(供上层展示/断言;不静默) */ +export interface AriaRecoveryReport { + /** 打开过程中被丢弃的 SSTable(按命名空间) */ + droppedSSTables: { namespace: string; id: number; level: number; reason: string }[]; + /** 是否怀疑已确认写入丢失(被丢弃的 SSTable 没有 WAL 兜底) */ + dataLossSuspected: boolean; + /** WAL 活跃区间内的分片空洞(已提交事务记录缺失) */ + walGaps: number[]; + /** 本次打开是否从旧格式(__aria_lsm_meta/__aria_schemas)迁移而来 */ + legacyImported: boolean; + /** 是否从更早的 manifest 世代回退(最新世代损坏) */ + manifestFallback: boolean; +} + // --------------------------------------------------------------------------- // AriaEngine // --------------------------------------------------------------------------- @@ -61,6 +81,38 @@ export class AriaEngine implements IStorageEngine { // v0.4.5: 多标签页独占锁(Web Locks API,OPFS 等无事务后端防并发写) private dbLock: DatabaseLock | null = null; + /** + * v0.8.0(B-6):存储层单一提交点。 + * + * 页面水位 / 各命名空间 SSTable 列表 / 表结构 / WAL 起始位置 / 待落盘冻结表意图 + * 全部收敛到 `__aria_manifest_`:数据先落盘,再提交 manifest, + * **提交成功后**才允许截断 WAL 或删除旧分片。恢复只认最后一份 CRC 通过的世代。 + */ + private manifestStore!: ManifestStore; + private manifest!: AriaManifest; + /** + * v0.8.0:已确认落盘的 WAL 水位(LSN)。 + * + * 只在"所有 LSM 都没有未落盘数据"时推进到当前 LSN —— 于是 + * `lsn <= durableLsn` 的记录必然已存在于已提交的 SSTable 中, + * 恢复时可以安全跳过(也就允许删除对应分片)。 + */ + private durableLsn = 0; + /** + * v0.8.0:仍需保留的最小 WAL 分片号(每次 `checkpointBefore` 的返回值)。 + * 写进 manifest 的 `wal.startSegment`:即便分片删除只完成一半,恢复也只从 + * 这个分片开始读,不会把上一世代的旧记录排到新记录之后重放。 + */ + private walStartSegment = 0; + /** v0.8.0:恢复诊断 */ + private recoveryReport: AriaRecoveryReport = { + droppedSSTables: [], + dataLossSuspected: false, + walGaps: [], + legacyImported: false, + manifestFallback: false, + }; + // 表结构 private schemas: Map = new Map(); private tablePKs: Map = new Map(); @@ -160,26 +212,37 @@ export class AriaEngine implements IStorageEngine { } await this.backend.open(dbName); - // 2a. FileManager (PageIO 实现) + Buffer Pool - this.fileManager = new FileManager(this.backend); - await this.fileManager.init(dbName); - this.bufferPool = new BufferPool(this.fileManager, this.config.bufferPoolPages); + // --------------------------------------------------------------------- + // 2. manifest(单一提交点):加载 → 旧格式迁移 → 认领所有权 + // --------------------------------------------------------------------- + this.manifestStore = new ManifestStore({ backend: this.backend }); + const loaded = await this.manifestStore.load(); + if (loaded.hadInvalidGenerations) { + // 有损坏世代被跳过:记录 + 告警(但仍可用更早的有效世代打开) + this.recoveryReport.manifestFallback = true; + // eslint-disable-next-line no-console + console.warn( + `[AriaEngine] manifest: skipped invalid generation(s): ` + + loaded.skipped.map((s) => `${s.generation} (${s.reason})`).join('; '), + ); + } + if (loaded.manifest === null) { + // 全新库或 v0.8.0 之前的旧格式库:先把旧格式状态**完整导入**, + // 再进行第一次提交 —— 顺序反了会写出"空 manifest"覆盖旧状态(静默空库)。 + this.manifest = createEmptyManifest({ instanceId: this.manifestStore.instanceId }); + this.manifestStore.adopt(this.manifest); + await this.importLegacyState(); + } else { + this.manifest = loaded.manifest; + } + // 认领所有权(世代 +1;允许接手别的实例 —— 之后旧实例的提交会被拒绝) + this.manifest = await this.manifestStore.claimOwnership(); + this.durableLsn = this.manifest.wal.startLsn; + this.walStartSegment = this.manifest.wal.startSegment; - // 2. 构建 SSTableStore - const sstableStore = this.createSSTableStore('main'); - - // 3. 初始化主 LSM(PK 索引) - this.lsm = new LSM({ - memtableSizeThreshold: this.config.memtableSizeThreshold, - levelSizeMultiplier: this.config.levelSizeMultiplier, - blockSize: this.config.pageSize, - bloomBitsPerKey: this.config.bloomFilterBitsPerKey, - // SSTable 缓存上限 = BufferPool 页数 × 页面大小(默认 256 页 ≈ 1MB 可控内存) - cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, - sstableStore, - }); - - // 4. 初始化 WAL + // --------------------------------------------------------------------- + // 3. WAL(LSN 从 manifest 高水位续接,全库单调) + // --------------------------------------------------------------------- // v0.4.5: 分片式 WAL 存储(__wal_%06d.bin),序号内嵌记录字节流无需 count 键, // append 单文件原子写;空洞检测截断;兼容旧格式 __wal_N + __wal_count this.wal = new WAL( @@ -187,28 +250,34 @@ export class AriaEngine implements IStorageEngine { this.config.walEnabled, this.config.walSyncMode, ); + this.wal.setLsn(this.manifest.wal.nextLsn); - // 5. 恢复 Schema + // --------------------------------------------------------------------- + // 4. FileManager (PageIO 实现) + Buffer Pool(页面水位下限来自 manifest) + // --------------------------------------------------------------------- + this.fileManager = new FileManager(this.backend); + await this.fileManager.init(dbName, this.manifest.pageIdWatermark); + this.bufferPool = new BufferPool(this.fileManager, this.config.bufferPoolPages); + + // --------------------------------------------------------------------- + // 5. 主 LSM(PK 索引)+ 二级索引 LSM + // --------------------------------------------------------------------- + this.lsm = this.createLSM('main'); + + // 5a. 恢复 Schema(manifest 权威;旧格式已在迁移阶段导入) await this.loadSchemas(); // v0.4.2-fix: 为 schema 中带 index/unique 标记的列重建二级索引 LSM。 // 此前重开只恢复 schema 不恢复索引 LSM → 索引查询静默回退全表、 // createIndex 因 colDef 已有标记直接 return → 索引永久缺失。 - // 索引数据已持久化在独立命名空间(sst_idx_* / meta),init() 直接加载。 + // 索引数据已持久化在独立命名空间(manifest.namespaces),init() 直接加载。 for (const [tableName, schema] of this.schemas) { const pkCol = this.tablePKs.get(tableName)!; for (const [colName, colDef] of Object.entries(schema.columns)) { if ((colDef.index || colDef.unique) && colName !== pkCol) { const idxKey = `${tableName}:idx:${colName}`; if (!this.secondaryIndexes.has(idxKey)) { - const idxLsm = new LSM({ - memtableSizeThreshold: this.config.memtableSizeThreshold, - levelSizeMultiplier: this.config.levelSizeMultiplier, - blockSize: this.config.pageSize, - bloomBitsPerKey: this.config.bloomFilterBitsPerKey, - cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, - sstableStore: this.createSSTableStore(`idx_${tableName}_${colName}`), - }); + const idxLsm = this.createLSM(`idx_${tableName}_${colName}`); await idxLsm.init(); this.secondaryIndexes.set(idxKey, idxLsm); } @@ -216,13 +285,32 @@ export class AriaEngine implements IStorageEngine { } } - // 6. 初始化 LSM(加载 SSTable 元数据) await this.lsm.init(); - // 7. WAL 恢复(两阶段:先扫描事务边界,仅回放已提交事务) + // --------------------------------------------------------------------- + // 6. WAL 恢复(两阶段:先扫描事务边界,仅回放已提交事务) + // --------------------------------------------------------------------- const committedTxns = new Set(); const allRecords: WALRecord[] = []; - await this.wal.recover((r) => allRecords.push(r)); + await this.wal.recover((r) => allRecords.push(r), { + // manifest 权威位置:小于 startLsn 的记录已确认落盘 → 跳过 + //(否则旧记录会把已删除的行复活);分片读取从 startSegment 起。 + fromSegment: this.manifest.wal.startSegment, + fromLsn: this.manifest.wal.startLsn, + // 空洞由引擎显式上报(恢复报告 + 告警),而不是静默丢弃尾部 + allowGaps: true, + }); + const walInfo = this.wal.getLastRecoveryInfo(); + if (walInfo && walInfo.gaps.length > 0) { + this.recoveryReport.walGaps = [...walInfo.gaps]; + this.recoveryReport.dataLossSuspected = true; + // eslint-disable-next-line no-console + console.warn( + `[AriaEngine] WAL segment gap detected: missing segment(s) ${walInfo.gaps.join(', ')} — ` + + 'transactions in the missing range are lost (recovery report records this)', + ); + } + // 第一遍:确定已提交事务 for (const r of allRecords) { if (r.type === WALRecordType.COMMIT) committedTxns.add(r.txnId); @@ -296,27 +384,50 @@ export class AriaEngine implements IStorageEngine { } // v0.3.3: 恢复完成后将回放数据落盘并截断 WAL, - // 避免每次重启重复回放 + WAL 无限膨胀 + // 避免每次重启重复回放 + WAL 无限膨胀。 + // + // v0.8.0(B-6)顺序固定为:**数据落盘 → manifest 提交 → 才允许截断 WAL**。 + // 每一步都有明确的失败语义: + // - flush 失败 → `ARIA_BACKGROUND_ERROR` 抛出(不截断 WAL,下次打开还能重放); + // - manifest 提交失败 → 同样不截断(回放数据仍可从 WAL 重建)。 + // 另外:二级索引数据**不在 WAL 里**(回放只写主 LSM),因此重建索引后必须 + // 先把索引 LSM 也落盘并提交,才能截断 WAL —— 否则崩溃后索引 memtable 丢失、 + // WAL 又为空(不触发重建)→ 索引静默变空(v0.4.2 的同类缺陷)。 if (allRecords.length > 0) { - await this.lsm.flush(); - await this.wal.checkpoint(); + await this.flushAllLsms(); // v0.4.2-fix: WAL 回放只更新主 LSM,二级索引 LSM 未同步 → // 崩溃前最后一批写入的索引缺失,重开时索引查询丢行。 // 恢复后全量重建所有表的二级索引(幂等)。 for (const tableName of this.schemas.keys()) { await this.reindexTableInternal(tableName); } + await this.flushAllLsms(); + await this.advanceWalCheckpoint(); + } else if (this.hasPendingFlushData()) { + // 没有回放记录但有未落盘数据(例如 manifest 已提交、内存态来自迁移): + // 同样走完整顺序,避免"内存有数据而 WAL 已被截断" + await this.flushAllLsms(); + await this.advanceWalCheckpoint(); } + // 7. 冻结表意图校验:manifest 声称有未落盘冻结表,但 WAL 已被截断/丢失 → + // 这些"已确认写入"真的没了,必须报错而不是安静地少一批行。 + this.verifyFrozenIntentsAfterRecovery(allRecords.length); + // 8. Checkpoint Manager(接入 WAL 大小阈值) // v0.4.2-fix: 事务活跃时 checkpoint 不得截断 WAL — // 否则 BEGIN/INSERT 记录被截断,COMMIT 后崩溃恢复丢失整个事务数据 + // + // v0.8.0(B-6/55):checkpoint 只等 **memtable 落盘**,不再等 compaction。 + // 此前 `lsm.flush()` 会排空整条链(含 compaction 级联),写路径每 1000 次操作 + // 就要等完整 compaction —— 这正是 v0.6.1 记录的"8~11s 悬崖"的另一半。 this.checkpointManager = new CheckpointManager( this.lsm, { checkpoint: async () => { if (this.currentTxnId) return; - await this.wal.checkpoint(); + // WAL 只在"数据已落盘 + manifest 已提交"之后才截断(见 advanceWalCheckpoint) + await this.advanceWalCheckpoint(); }, flush: async () => { if (this.currentTxnId) return; @@ -330,10 +441,11 @@ export class AriaEngine implements IStorageEngine { // v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM — // 此前只 flush 主 LSM,checkpoint 截断 WAL 后崩溃时索引 memtable 未落盘、 // WAL 为空跳过重建 → 二级索引静默丢失最后一批条目(生产数据一致性问题) - await this.lsm.flush(); - for (const idxLsm of this.secondaryIndexes.values()) { - await idxLsm.flush(); - } + await this.flushAllLsms(); + }, + // v0.8.0(B-6/55):周期 checkpoint 只落 memtable,不等 compaction + flushMemtables: async () => { + await this.flushAllLsms(true); }, } as any, this.config.checkpointInterval, @@ -345,64 +457,88 @@ export class AriaEngine implements IStorageEngine { async close(): Promise { if (!this.opened) return; - await this.persistSchemas(); - await this.lsm.flush(); - // v0.4.2-fix: 同步落盘全部二级索引 LSM — 此前只 flush 主 LSM, - // 优雅关闭后索引 memtable 未落盘 → 重开索引为空 → 索引查询返回空结果 - for (const idxLsm of this.secondaryIndexes.values()) { - await idxLsm.flush(); + // v0.8.0(B-6):关闭顺序 = 全部数据落盘 → manifest 提交 → 才截断 WAL。 + // 每一步失败都不会导致"WAL 被截断而数据没落盘"。 + // + // v0.8.0:整体加 try/finally —— 落盘失败(如介质故障)时**仍然要**关闭后端、 + // 释放 Web Locks 并清空运行期状态,然后把错误抛给调用方。修复前只要 flush + // 抛错,后面每一步都不会执行:库锁永久占着、后端不关、内存状态残留 + //(计划"生命周期与 API 面"清单里的 `close()` 无 try/finally 项)。 + let failure: unknown = null; + try { + await this.flushAllLsms(); + // v0.4.5: 页面化存储 — 落盘全部脏页(save 已逐页落盘,此处兜底) + await this.bufferPool.flushAll(); + await this.advanceWalCheckpoint(); + } catch (error) { + failure = error; + } finally { + try { + await this.backend.close(); + } catch (error) { + if (failure === null) failure = error; + } + // v0.4.5: 释放多标签页独占锁(等待锁真正归还) + if (this.dbLock) { + try { + await this.dbLock.release(); + } catch { /* 锁释放失败不覆盖已有错误 */ } + this.dbLock = null; + } + // v0.4.2-fix: 清空运行期状态(此前 close 后 mvcc/txn 残留, + // 重开时 beginTransaction 报 TX_ACTIVE 或读到陈旧快照) + this.schemas.clear(); + this.tablePKs.clear(); + this.secondaryIndexes.clear(); + this.uniqueIndexCols.clear(); + this.mvcc = new MVCCManager(); + this.currentTxnId = null; + this.txnSnapshot = null; + this.savepoints.clear(); + this.opCounter = 0; + this.opened = false; } - // v0.4.5: 页面化存储 — 落盘全部脏页(save 已逐页落盘,此处兜底) - await this.bufferPool.flushAll(); - await this.wal.flush(); - // v0.4.2-fix: close 前 checkpoint(截断 WAL)— - // 此前只 flush 不截断,下次打开会重放全部历史 WAL 记录(含已落盘 SSTable 的数据), - // 重复解析/重复 put 拖慢启动,并与恢复后 flush+checkpoint 竞争放大丢数据 - await this.wal.checkpoint(); - await this.backend.close(); - // v0.4.5: 释放多标签页独占锁(等待锁真正归还) - if (this.dbLock) { - await this.dbLock.release(); - this.dbLock = null; - } - // v0.4.2-fix: 清空运行期状态(此前 close 后 mvcc/txn 残留, - // 重开时 beginTransaction 报 TX_ACTIVE 或读到陈旧快照) - this.schemas.clear(); - this.tablePKs.clear(); - this.secondaryIndexes.clear(); - this.uniqueIndexCols.clear(); - this.mvcc = new MVCCManager(); - this.currentTxnId = null; - this.txnSnapshot = null; - this.savepoints.clear(); - this.opCounter = 0; - this.opened = false; + if (failure !== null) throw failure; } /** * v0.4.2-fix: 崩溃恢复/自愈 — 校验并移除损坏 SSTable、截断 WAL、重建二级索引。 * v0.4.5 增强:清理 OPFS 残留临时文件、清理孤儿页面(meta 未引用的 pg_ 文件)。 * 应用层检测到异常后调用,无需删库重建。 + * + * v0.8.0(B-6):孤儿回收**必须**以"manifest 健康"为前提。 + * 修复前 `cleanupOrphanPages` 只读裸 JSON meta,读不出来就当"没有任何引用", + * 于是元数据损坏时 repair 会把全部活页删掉(不可逆)。现在: + * - 只要本次打开出现过损坏世代/被丢弃的 SSTable → 直接跳过回收并告警; + * - 引用集合来自 manifest 的权威 meta 列表。 */ async repair(): Promise { this.ensureOpen(); // v0.6.0-fix: 先清页面缓存再校验 — 缓存中的"完好页面"会掩盖磁盘损坏 await this.bufferPool.clear(); // 1. 校验全部 SSTable,移除残缺项(打开时已做一次,此处兜底运行期损坏) - const removed = await this.lsm.validateAll(); - // 2. 将 WAL 残留数据落盘并截断,避免无限重放(含空洞截断落地) - await this.lsm.flush(); - await this.wal.checkpoint(); + let removed = 0; + for (const lsm of this.allLsms()) { + removed += await lsm.validateAll(); + } + // 2. 将 WAL 残留数据落盘并提交,之后才允许截断 + await this.flushAllLsms(); + await this.advanceWalCheckpoint(); // 3. 重建所有表的二级索引(修复索引与主数据不一致) for (const tableName of this.schemas.keys()) { await this.reindexTable(tableName); } + await this.flushAllLsms(); + await this.commitManifest(); + // v0.8.0(B-6):此时两条链都已静默 → 没有在途读者 → 强制回收退休文件, + // 避免它们被下一步的孤儿页回收"顺带"删掉(结果相同,但状态更干净) + for (const lsm of this.allLsms()) lsm.reclaimRetiredNow(); // v0.4.5: 4. 清理 OPFS 残留临时文件(.crswap/.tmp) const backendAny = this.backend as unknown as { cleanupStaleFiles?: () => Promise }; if (typeof backendAny.cleanupStaleFiles === 'function') { try { await backendAny.cleanupStaleFiles(); } catch { /* 清理失败不阻塞 */ } } - // v0.4.5: 5. 清理孤儿页面(所有 LSM 命名空间 meta 均未引用的 pg_ 文件) + // v0.4.5: 5. 清理孤儿页面(manifest 全部命名空间均未引用的 pg_ 文件) await this.cleanupOrphanPages(); if (removed > 0) { // eslint-disable-next-line no-console @@ -411,35 +547,39 @@ export class AriaEngine implements IStorageEngine { } /** - * v0.4.5: 清理孤儿页面 — 扫描全部 pg_* 文件,未被任何 LSM 命名空间 meta 引用的删除。 + * v0.4.5: 清理孤儿页面 — 扫描全部 pg_* 文件,未被任何命名空间 meta 引用的删除。 * 孤儿页面来自:崩溃中断的 compaction/删除流程(旧 SSTable 页面残留)。 + * + * v0.8.0(B-6):引用集合取自 manifest;且**只有在没有损坏迹象时才执行** —— + * "任何引用不到的东西一律保留而非删除"在恢复路径上是不变量,只有显式 repair + * 且 manifest 完整可信时才允许回收空间。 */ private async cleanupOrphanPages(): Promise { + if (this.recoveryReport.dataLossSuspected || this.recoveryReport.manifestFallback) { + // eslint-disable-next-line no-console + console.warn( + '[AriaEngine] repair: skipping orphan-page reclamation — recovery report shows ' + + 'damage or manifest fallback (unreferenced pages are kept, never deleted blindly)', + ); + return; + } const keys = await this.backend.listKeys(); const pgKeys = keys.filter((k) => /^pg_\d+$/.test(k)); if (pgKeys.length === 0) return; const used = new Set(); - const collectMeta = async (ns: string): Promise => { - const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`; - const raw = await this.backend.read(META_KEY); - if (!raw) return; - try { - const metas = JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[]; - for (const m of metas) { - if (m.pageIds) { - for (const pid of m.pageIds) used.add(pid); - } + for (const state of Object.values(this.manifest.namespaces)) { + for (const m of state.sstables) { + if (m.pageIds) { + for (const pid of m.pageIds) used.add(pid); } - } catch { /* 损坏的 meta 忽略(validateAll 已处理) */ } - }; - - await collectMeta('main'); - // 收集全部二级索引命名空间 - for (const [tableName, schema] of this.schemas) { - for (const [colName, colDef] of Object.entries(schema.columns)) { - if (colDef.index || colDef.unique) { - await collectMeta(`idx_${tableName}_${colName}`); + } + } + // 未落盘的页面(正在写入的 SSTable)也不能删 + for (const lsm of this.allLsms()) { + for (const level of this.getLsmLevels(lsm)) { + for (const meta of level) { + if (meta.pageIds) for (const pid of meta.pageIds) used.add(pid); } } } @@ -449,9 +589,17 @@ export class AriaEngine implements IStorageEngine { .filter((pid) => !used.has(pid)); if (orphanIds.length > 0) { await this.backend.deleteMany(orphanIds.map((pid) => `pg_${pid}`)); + // eslint-disable-next-line no-console + console.warn(`[AriaEngine] repair: reclaimed ${orphanIds.length} orphan page file(s)`); } } + /** v0.8.0: 读取某个 LSM 当前引用的层结构(诊断/孤儿回收用) */ + private getLsmLevels(lsm: LSM): SSTableMeta[][] { + return (lsm as unknown as { levels: SSTableMeta[][] }).levels ?? []; + } + + /** * v0.4.1: 重置数据库 — 清空全部数据与表结构(演示页刷新/重新初始化用)。 * 清空存储后端、LSM、WAL、MVCC 与二级索引,后续可继续使用本实例。 @@ -476,6 +624,17 @@ export class AriaEngine implements IStorageEngine { // 持久化空 schema(防止旧 schema 记录残留) await this.persistSchemas(); // 重置 WAL 状态(backend.clear 已清记录,同步内存计数) + // v0.8.0(B-6):清空后 manifest 里不能再残留任何 meta —— 否则重开会引用 + // 已被删除的页面(幽灵数据)。这里把命名空间状态整表清掉并提交。 + this.manifest.namespaces = {}; + this.manifest.frozen = []; + this.manifest.pageIdWatermark = Math.max(this.manifest.pageIdWatermark, this.fileManager.getNextPageId()); + this.durableLsn = this.wal.getLsn(); + this.walStartSegment = 0; + // 整库清空:介质被整体抹掉、manifest 也重新从 0 开始 —— 此时才允许 + // 把 WAL 分片编号也重置(其它任何路径都不允许复用分片号) + this.wal.reset(); + await this.commitManifest(); await this.wal.checkpoint(); } @@ -665,7 +824,6 @@ export class AriaEngine implements IStorageEngine { validatedRows.push({ row: validated, pkValue, key: `${tableName}:${pkValue}` }); } - await this.lsm.prefetchKeys(validatedRows.map((v) => v.key)); // v0.7.3: 主键批内互查 + 预检 —— 此前 PK 重复检查在写入循环内: // 第 N 行重复抛错时,前 N-1 行已 put LSM 且其 WAL 记录随 appendBatch 一起 // 丢失 → 语句级部分提交 + 内存/WAL 不一致(与 v0.6.2 的 unique 预检同一阶段)。 @@ -689,20 +847,12 @@ export class AriaEngine implements IStorageEngine { ); } } - // v0.6.2: 唯一约束 — 批量预加载本批唯一列涉及的索引范围(一次 drainChain) - for (const colName of uniqueCols) { - const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`)!; - await idxLsm.prefetchPrefixRanges( - validatedRows - .map((v): [string, string] | null => { - const val = v.row[colName]; - if (val === undefined || val === null) return null; - const p = `${String(val)}:`; - return [p, `${p}\uffff`]; - }) - .filter((r): r is [string, string] => r !== null), - ); - } + // v0.8.0(B-6/55):唯一约束预检不再需要"批量预加载索引范围"。 + // + // 修复前这里要遍历本批所有唯一列值、把涉及的范围全部 prefetch 进缓存, + // 只为满足"读之前必须先把数据读进缓存"这条隐式约定(LSM 缓存未命中会 + // 静默跳过整个 SSTable)。现在读取自洽(未命中即回源 + CRC 校验), + // 预加载这一步连同它带来的 drainChain 等待一起删除。 // v0.6.2: 唯一性整批预检(批内互查 + 索引查)—— 失败整批不落库(原子语义) const batchUnique = new Map>(); for (const { row: validated, pkValue } of validatedRows) { @@ -843,24 +993,8 @@ export class AriaEngine implements IStorageEngine { } } - // v0.6.2: 唯一约束 — 批量预加载本批更新涉及的唯一列索引范围(一次 drainChain) + // v0.8.0(B-6/55):唯一约束检查不再需要预加载索引范围(读取自洽,见 insert) const uniqueCols = this.uniqueColumns(tableName, schema); - for (const colName of uniqueCols) { - const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`)!; - const ranges: [string, string][] = []; - if (cleanUpdates[colName] !== undefined && cleanUpdates[colName] !== null) { - const p = `${String(cleanUpdates[colName])}:`; - ranges.push([p, `${p}\uffff`]); - } else if (!(colName in cleanUpdates)) { - for (const row of rows) { - const val = row[colName]; - if (val === undefined || val === null) continue; - const p = `${String(val)}:`; - ranges.push([p, `${p}\uffff`]); - } - } - await idxLsm.prefetchPrefixRanges(ranges); - } // v0.7.2: 语句级原子性 — 两阶段(先全量预检,后执行)。 // 此前逐行"校验+写入":第 N 行唯一冲突/校验失败抛错时,前 N-1 行已写入 @@ -1364,7 +1498,6 @@ export class AriaEngine implements IStorageEngine { // 全表惰性扫描(含 WHERE 过滤,不物化;v0.7.4: callback 返回 false 提前终止, // 未消费的 SSTable 块 / 子树不再解析 —— 真流式,大表 limit 内存 O(1)) - await this.lsm.prefetchRange(prefix, `${prefix}\uffff`); await this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => { if (count >= limit) return false; const row = { ...value }; @@ -1509,7 +1642,6 @@ export class AriaEngine implements IStorageEngine { // 重写主 LSM:移除所有行的该列键(find 副本无法就地删除,必须重写存储) const prefix = `${tableName}:`; const endKey = `${prefix}\uffff`; - await this.lsm.prefetchRange(prefix, endKey); const entries = await this.lsm.rangeScan(prefix, endKey); const walRecords: Omit[] = []; for (const [key, value] of entries) { @@ -1848,7 +1980,6 @@ export class AriaEngine implements IStorageEngine { const pkCol = this.tablePKs.get(tableName)!; const prefix = `${tableName}:`; // 预加载范围内涉及的 SSTable,避免 rangeScan 时缓存未命中静默丢数据 - await this.lsm.prefetchRange(prefix, `${prefix}\uffff`); const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`); const rows = entries.map(([key, value]) => { // v0.8.0: 深拷贝(此前 `{ ...value }` 只做浅拷贝,嵌套 json 值仍与 LSM @@ -1922,51 +2053,363 @@ export class AriaEngine implements IStorageEngine { // Schema 持久化 // ======================================================================= + /** + * v0.8.0(B-6):表结构**随 manifest 一起提交**(单一提交点)。 + * + * 修复前 DDL 结束时会单独写一份 `__aria_schemas`:结构变更与存储状态 + * (SSTable meta / 页面 / WAL 水位)各自落盘,中间崩溃就会留下"结构说加过列、 + * 数据里没有"或反之的分裂状态。现在两者在同一次原子提交里生效。 + */ private async persistSchemas(): Promise { + await this.commitManifest(); + } + + private async loadSchemas(): Promise { + // 权威来源:manifest(旧格式已在 importLegacyState 阶段导入) + let data = this.manifest.schemas ?? {}; + + // 兼容回退:manifest 中没有任何表结构,但旧格式记录存在 + //(例如上一次是被旧版本打开的库)→ 读进来并纳入 manifest + if (Object.keys(data).length === 0) { + const raw = await this.backend.read('__aria_schemas'); + if (raw) { + try { + const parsed = JSON.parse(new TextDecoder().decode(raw)) as Record>; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + data = parsed; + this.manifest.schemas = parsed; + } + } catch (error) { + // 修复前这里 `catch {}` 静默忽略 → 坏 schema = 看不到任何表(静默空库) + throw new DatabaseError( + `Schema record is corrupt and cannot be loaded: ${(error as Error).message}`, + 'ARIA_LEGACY_META_CORRUPT', + error, + ); + } + } + } + + for (const [tableName, columns] of Object.entries(data)) { + const schema: TableSchema = { name: tableName, columns }; + this.schemas.set(tableName, schema); + this.tablePKs.set(tableName, this.getPK(schema)); + } + } + + + // ======================================================================= + // v0.8.0(B-6):单一提交点与命名空间工厂 + // ======================================================================= + + /** + * v0.8.0: 恢复诊断(打开时被丢弃的 SSTable、WAL 空洞、是否怀疑数据丢失)。 + * + * 数据来自两处:引擎层(WAL 空洞 / 迁移 / manifest 回退)与各 LSM(被丢弃的 + * SSTable + 其 `dataLossSuspected`),这里合并成一份对外的报告 —— 于是 + * "这次打开到底自愈了什么、有没有真丢数据"是**可读的返回值**而不是只能翻日志。 + */ + getRecoveryReport(): AriaRecoveryReport { + const dropped = this.recoveryReport.droppedSSTables.map((d) => ({ ...d })); + let dataLoss = this.recoveryReport.dataLossSuspected; + for (const lsm of this.allLsms()) { + const r = lsm.getRecoveryReport(); + for (const d of r.droppedSSTables) { + dropped.push({ namespace: r.namespace, ...d }); + } + if (r.dataLossSuspected) dataLoss = true; + } + return { + droppedSSTables: dropped, + dataLossSuspected: dataLoss, + walGaps: [...this.recoveryReport.walGaps], + legacyImported: this.recoveryReport.legacyImported, + manifestFallback: this.recoveryReport.manifestFallback, + }; + } + + /** + * v0.8.0: LSM 的唯一构造点。 + * + * 为什么集中:本项目最反复的缺陷模式就是"同一语义在多处实现、只修一处" + * (审计结论的原话)。主 LSM 与二级索引 LSM 的配置必须完全一致地带上 + * 命名空间、WAL LSN 提供者与 durable-coverage 语义,因此只能有一个工厂。 + */ + private createLSM(ns: string): LSM { + return new LSM({ + memtableSizeThreshold: this.config.memtableSizeThreshold, + levelSizeMultiplier: this.config.levelSizeMultiplier, + blockSize: this.config.pageSize, + bloomBitsPerKey: this.config.bloomFilterBitsPerKey, + // SSTable 缓存上限 = BufferPool 页数 × 页面大小(默认 256 页 ≈ 1MB 可控内存) + cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, + sstableStore: this.createSSTableStore(ns), + namespace: ns, + // 冻结时刻的 WAL LSN → manifest 的冻结表意图(WAL 水位下限) + walLsnProvider: () => this.wal?.getLsn() ?? 0, + // WAL 已被截断(startLsn > 0)时,SSTable 缺失/损坏 = 已确认数据真的丢了 + requireDurableCoverage: (this.manifest?.wal.startLsn ?? 0) > 0, + }); + } + + /** 全部 LSM(主 + 二级索引) */ + private allLsms(): LSM[] { + return [this.lsm, ...this.secondaryIndexes.values()]; + } + + /** + * v0.8.0: 把全部 LSM 的数据落盘。 + * @param memtablesOnly true = 只落 memtable,不等 compaction(checkpoint 用) + */ + private async flushAllLsms(memtablesOnly: boolean = false): Promise { + for (const lsm of this.allLsms()) { + if (memtablesOnly) await lsm.flushMemtablesOnly(); + else await lsm.flush(); + } + } + + /** v0.8.0: 是否存在任何未落盘的 LSM 数据(决定 WAL 水位能否推进) */ + private hasPendingFlushData(): boolean { + return this.allLsms().some((lsm) => lsm.hasPendingFlushData()); + } + + /** schema 的持久化形态(与旧 `__aria_schemas` 同形) */ + private serializeSchemas(): Record> { const data: Record> = {}; for (const [name, schema] of this.schemas) { data[name] = schema.columns; } - const json = JSON.stringify(data); - const buf = new TextEncoder().encode(json).buffer; - await this.backend.write('__aria_schemas', buf); + return data; } - private async loadSchemas(): Promise { - const raw = await this.backend.read('__aria_schemas'); - if (!raw) return; + /** 所有 LSM 的待落盘冻结表意图(manifest 记录,阻止 WAL 水位越过它们) */ + private collectFrozenIntents(): ManifestFrozenIntent[] { + const intents: ManifestFrozenIntent[] = []; + for (const lsm of this.allLsms()) { + intents.push(...lsm.getFrozenIntents()); + } + return intents; + } - try { - const json = new TextDecoder().decode(raw); - const data = JSON.parse(json) as Record>; + /** + * v0.8.0(B-6):**唯一的 manifest 提交入口**。 + * + * 每次提交都重新计算权威字段,因此并发/交错场景下"最后一次提交"总是包含 + * 完整的最新状态: + * - `pageIdWatermark`:单调推进、永不复用; + * - `schemas`:表结构的权威描述(DDL 不再依赖独立的 `__aria_schemas` 提交); + * - `frozen`:待落盘冻结表意图; + * - `wal.startLsn`:**只有全部数据已落盘时才推进**(否则保持原值), + * 这是"截断 WAL 前必须先提交 manifest"的可验证形式。 + */ + private async commitManifest(targetDurableLsn?: number): Promise { + if (!this.opened && !this.manifestStore) return; + const intents = this.collectFrozenIntents(); + this.manifest.pageIdWatermark = Math.max( + this.manifest.pageIdWatermark, + this.fileManager.getNextPageId(), + ); + this.manifest.schemas = this.serializeSchemas(); + this.manifest.frozen = intents; - for (const [tableName, columns] of Object.entries(data)) { - const schema: TableSchema = { name: tableName, columns }; - this.schemas.set(tableName, schema); - this.tablePKs.set(tableName, this.getPK(schema)); + const computed = this.computeDurableLsn(intents); + // 允许调用方给一个**更保守**的目标水位(如"先算分片边界、再提交"的清理路径)。 + // 只取更小者:水位永远不允许超过"当前真实可保证"的值。 + this.durableLsn = typeof targetDurableLsn === 'number' + ? Math.min(computed, Math.max(this.durableLsn, targetDurableLsn)) + : computed; + + this.manifest.wal = { + startSegment: this.walStartSegment, + startLsn: this.durableLsn, + nextLsn: Math.max(this.manifest.wal.nextLsn, this.wal.getLsn()), + }; + this.manifest = await this.manifestStore.commit(); + } + + /** + * v0.8.0:计算"当前可以保证的落盘水位"(纯函数,不改状态)。 + * + * 三条规则: + * - 有冻结表意图 → 水位不得越过最早的冻结表内容起点(它们的记录只在内存+WAL); + * - 有未落盘 memtable → 维持原水位; + * - 全部落盘 → 推进到当前 LSN(这些记录已存在于已提交的 SSTable 中)。 + */ + private computeDurableLsn(intents: ManifestFrozenIntent[]): number { + if (intents.length > 0) { + return Math.min(this.durableLsn, Math.min(...intents.map((i) => i.lsnAtFreeze))); + } + if (this.hasPendingFlushData()) return this.durableLsn; + return Math.max(this.durableLsn, this.wal.getLsn()); + } + + /** + * v0.8.0(B-6):WAL 检查点的**唯一实现** —— "先算边界 → 提交 manifest → 再删除"。 + * + * 顺序不可交换(见 `SegmentedWALStore.planKeepFrom` 的说明)。所有需要回收 WAL + * 空间的路径(打开恢复后、close、repair、周期 checkpoint)都必须走这里, + * 否则又会出现"同一语义多处实现、只改一处"的老问题。 + */ + private async advanceWalCheckpoint(): Promise { + await this.wal.flush(); + // 1. 先算:以"如果没有未落盘数据,水位会到哪里"为基准 + const target = this.hasPendingFlushData() + ? this.durableLsn + : Math.max(this.durableLsn, this.wal.getLsn()); + // 2. 再提交(manifest 记录 startSegment + startLsn) + // 注意:先算边界是为了让 manifest 里的 startSegment **不小于**介质上真实存在的 + // 分片号 —— 否则恢复时无法区分"正常清理过的前缀"与"介质丢了一段记录"。 + this.walStartSegment = await this.wal.planKeepFromSegment(target); + await this.commitManifest(target); + // 3. 最后才允许删除分片 + await this.wal.checkpointBefore(target); + } + + /** + * v0.8.0:旧格式(v0.8.0 之前)状态导入。 + * + * 导入必须是**全有或全无**的: + * - 旧 meta 存在但无法解析 → 抛错(`ARIA_LEGACY_META_CORRUPT`)。 + * 修复前 `readMetaList()` 遇到坏 JSON 返回 `[]`,于是"元数据损坏"直接 + * 表现为"空库",随后 repair 还会把没人引用的活页全部删掉(不可逆)。 + */ + private async importLegacyState(): Promise { + const keys = await this.backend.listKeys(); + let imported = false; + + // 1. 各命名空间 SSTable meta(__aria_lsm_meta / __aria_lsm_meta_) + const metaKeys = keys.filter((k) => k === '__aria_lsm_meta' || k.startsWith('__aria_lsm_meta_')); + for (const key of metaKeys) { + const ns = key === '__aria_lsm_meta' ? 'main' : key.slice('__aria_lsm_meta_'.length); + const raw = await this.backend.read(key); + if (!raw) continue; + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(raw)); + } catch (error) { + throw new DatabaseError( + `Legacy SSTable metadata "${key}" is corrupt and cannot be migrated: ${(error as Error).message}`, + 'ARIA_LEGACY_META_CORRUPT', + error, + ); } - } catch { - // 忽略损坏的 schema 数据 + if (!Array.isArray(parsed)) { + throw new DatabaseError( + `Legacy SSTable metadata "${key}" is not an array`, + 'ARIA_LEGACY_META_CORRUPT', + ); + } + const sstables: SSTableMeta[] = []; + for (const item of parsed) { + const m = item as Partial; + if (typeof m?.id !== 'number' || typeof m?.level !== 'number' + || typeof m?.minKey !== 'string' || typeof m?.maxKey !== 'string') { + throw new DatabaseError( + `Legacy SSTable metadata "${key}" has an entry with unexpected shape: ${JSON.stringify(item)}`, + 'ARIA_LEGACY_META_CORRUPT', + ); + } + sstables.push({ + id: m.id, + level: m.level, + minKey: m.minKey, + maxKey: m.maxKey, + blockCount: typeof m.blockCount === 'number' ? m.blockCount : 0, + totalSize: typeof m.totalSize === 'number' ? m.totalSize : 0, + bloomData: null, + ...(Array.isArray(m.pageIds) ? { pageIds: [...m.pageIds] } : {}), + }); + } + sstables.sort((a, b) => b.id - a.id); + this.manifest.namespaces[ns] = { + nextSstableId: sstables.reduce((max, m) => Math.max(max, m.id), 0), + sstables, + }; + imported = true; + } + + // 2. 表结构(__aria_schemas) + const schemaRaw = await this.backend.read('__aria_schemas'); + if (schemaRaw) { + try { + const parsed = JSON.parse(new TextDecoder().decode(schemaRaw)) as Record>; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + this.manifest.schemas = parsed; + imported = true; + } + } catch (error) { + throw new DatabaseError( + `Legacy schema record "__aria_schemas" is corrupt and cannot be migrated: ${(error as Error).message}`, + 'ARIA_LEGACY_META_CORRUPT', + error, + ); + } + } + + // 3. 页面水位(__aria_meta,仅作为单调下限) + const pageMeta = await this.backend.read('__aria_meta'); + if (pageMeta && pageMeta instanceof ArrayBuffer && pageMeta.byteLength >= 4) { + const watermark = new DataView(pageMeta).getUint32(0, false); + if (watermark > this.manifest.pageIdWatermark) { + this.manifest.pageIdWatermark = watermark; + imported = true; + } + } + + if (imported) { + this.recoveryReport.legacyImported = true; + // eslint-disable-next-line no-console + console.warn( + '[AriaEngine] migrated legacy storage layout into __aria_manifest ' + + '(old keys are kept untouched as a fallback)', + ); } } - // ======================================================================= - // SSTableStore 构建 - // ======================================================================= + /** + * v0.8.0:恢复后校验冻结表意图。 + * + * 语义:manifest 记录了"某张冻结表还没落盘"(意图),说明它的数据要么在 WAL 里, + * 要么已经在 SSTable 里。若本次打开**一条 WAL 记录都没有重放到**,而 manifest + * 又声称有未落盘数据,那么这些"已确认写入"就是真的丢了(WAL 被截断/介质丢失)。 + * 此时抛错 —— 修复前这种丢失完全不可观测。 + */ + private verifyFrozenIntentsAfterRecovery(replayedRecordCount: number): void { + const intents = this.manifest.frozen; + if (intents.length === 0) return; + if (replayedRecordCount > 0) return; // WAL 覆盖到了这些数据(重放会重建) + if (!this.config.walEnabled) return; // 未启用 WAL:本来就没有日志兜底(配置语义) + // 全部 LSM 已落盘(意图来自上一次会话的残留)→ 数据其实已经安全 + if (!this.hasPendingFlushData() && this.manifest.frozen.length === 0) return; + + const summary = intents.map((i) => `${i.ns}#${i.id}(${i.entryCount} 项)`).join(', '); + throw new DatabaseError( + `AriaEngine manifest declares ${intents.length} un-flushed frozen table(s) [${summary}] ` + + 'but no WAL record was replayed — confirmed writes are missing ' + + '(WAL truncated or lost, and the data is not in any committed SSTable)', + 'ARIA_WRITE_LOST', + ); + } /** - * 创建命名空间隔离的 SSTableStore。 + * 创建命名空间隔离的 SSTableStore(**manifest 权威**)。 * * 主 LSM 与每个二级索引 LSM 各持有独立实例: * - 文件 key 前缀隔离(sst_ / sst_idx_${table}_${col}_) - * - 元数据 key 隔离(__aria_lsm_meta / __aria_lsm_meta_${ns}) - * - id 序列独立(避免 v0.2.4 共享 id 空间导致的文件互相覆盖) + * - 元数据在 manifest 的 `namespaces[ns]` 中隔离(不再各写一份裸 JSON) + * - id 序列独立且**单调推进**(`nextSstableId` 记在 manifest 里, + * 即便某一代 SSTable 全部被删除,id 也不会被复用) + * + * v0.8.0(B-6)两处结构变化: + * 1. **meta 不再走裸 JSON**:`saveMeta`/`deleteMeta` 直接改 manifest 并提交。 + * 修复前 `JSON.parse` 失败 → 返回 `[]` → 元数据损坏 = 静默空库 + * (随后 repair 还会把没人引用的活页删掉); + * 2. `load/delete` 的页面映射改由 PageSSTableStore 自己维护 + * (活跃 + 退休两张表)—— 于是"compaction 摘除 meta"与 + * "在途读者按 id 读取"不再互相矛盾。 */ private createSSTableStore(ns: string): SSTableStore { const filePrefix = ns === 'main' ? 'sst_' : `sst_${ns}_`; - const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`; - let seq = 0; - let seqLoaded = false; // v0.4.5: 页面化物理存储(OPFS 后端默认启用)— SSTable 存为 4KB 页面,BufferPool 缓存 const usePages = this.isPageStorage(); @@ -1976,18 +2419,24 @@ export class AriaEngine implements IStorageEngine { ? new PageSSTableStore(this.fileManager, this.bufferPool, this.config.compression) : null; - const encodeText = (text: string): ArrayBuffer => { - return new TextEncoder().encode(text).buffer; - }; - - const readMetaList = async (): Promise => { - const raw = await this.backend.read(META_KEY); - if (!raw) return []; - try { - return JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[]; - } catch { - return []; + // 打开时把 manifest 中已有的页面映射注册进 pageStore, + // 使 load(id) 不再依赖"调用方传 pageIds"(退休 SSTable 也要能读) + if (pageStore) { + const existing = this.manifest.namespaces[ns]?.sstables ?? []; + for (const meta of existing) { + if (meta.pageIds && meta.pageIds.length > 0) { + pageStore.registerPageIds(meta.id, meta.pageIds, meta.totalSize); + } } + } + + const nsState = (): { nextSstableId: number; sstables: SSTableMeta[] } => { + let state = this.manifest.namespaces[ns]; + if (!state) { + state = { nextSstableId: 0, sstables: [] }; + this.manifest.namespaces[ns] = state; + } + return state; }; return { @@ -2008,59 +2457,65 @@ export class AriaEngine implements IStorageEngine { return { storedSize: buf.byteLength }; }, load: async (id) => { - // 页面化读取:meta 有 pageIds → 页面拼接;无(旧数据)→ 整 value + // 页面化读取:pageStore 自己维护 id → pageIds(含退休表) if (pageStore) { - const metas = await readMetaList(); - const meta = metas.find((m) => m.id === id); - if (meta && meta.pageIds && meta.pageIds.length > 0) { - return pageStore.load(id, meta.pageIds, meta.totalSize); - } + const paged = await pageStore.load(id); + if (paged !== null) return paged; } + // 旧数据(页面化之前写入的整 value)或页面缺失 → 回退整 value 读取 const raw = await this.backend.read(`${filePrefix}${id}`); if (!raw) return null; let buf = new Uint8Array(raw); // 解压(若启用)— 解密由 EncryptedBackend 在 backend 层透明处理(v0.4.5) - if (this.config.compression) { + if (this.config.compression && !pageStore) { // v0.4.5: 压缩流自带原始大小头,无需外部估算 buf = decompressLZ4(buf) as Uint8Array; } return buf; }, delete: async (id) => { - if (pageStore) { - const metas = await readMetaList(); - const meta = metas.find((m) => m.id === id); - if (meta && meta.pageIds && meta.pageIds.length > 0) { - await pageStore.delete(id, meta.pageIds); - } - } + if (pageStore) await pageStore.delete(id); await this.backend.delete(`${filePrefix}${id}`); }, allocateId: async () => { - // 从本命名空间的 meta 恢复 id 序列,保证单调递增且不与其他 LSM 冲突 - if (!seqLoaded) { - const metas = await readMetaList(); - seq = metas.reduce((m, x) => Math.max(m, x.id), 0); - seqLoaded = true; - } - return ++seq; + // 单调推进、永不复用:id 序列记在 manifest 里(不依赖"现存最大 id") + const state = nsState(); + const maxExisting = state.sstables.reduce((max, m) => Math.max(max, m.id), 0); + state.nextSstableId = Math.max(state.nextSstableId, maxExisting) + 1; + return state.nextSstableId; }, - listMeta: readMetaList, + listMeta: async () => nsState().sstables.map((m) => ({ ...m })), saveMeta: async (meta) => { - const list = await readMetaList(); + const state = nsState(); // v0.4.5: 页面化时把页面 ID 列表注入 meta(save 后、saveMeta 前由 LSM 顺序调用) const pageIds = pageStore?.getPageIds(meta.id); - const metaWithPages = pageIds && pageIds.length > 0 ? { ...meta, pageIds } : meta; - // 更新或添加 - const idx = list.findIndex((m) => m.id === meta.id); - if (idx >= 0) list[idx] = metaWithPages; - else list.push(metaWithPages); - await this.backend.write(META_KEY, encodeText(JSON.stringify(list))); + const metaWithPages: SSTableMeta = pageIds && pageIds.length > 0 + ? { ...meta, pageIds: [...pageIds] } + : { ...meta, bloomData: null }; + const idx = state.sstables.findIndex((m) => m.id === meta.id); + if (idx >= 0) state.sstables[idx] = metaWithPages; + else state.sstables.push(metaWithPages); + state.nextSstableId = Math.max(state.nextSstableId, meta.id); + // ★ 单一提交点:数据已落盘(save 返回)→ 现在提交元数据 + await this.commitManifest(); }, deleteMeta: async (id) => { - const list = await readMetaList(); - const filtered = list.filter((m) => m.id !== id); - await this.backend.write(META_KEY, encodeText(JSON.stringify(filtered))); + const state = nsState(); + state.sstables = state.sstables.filter((m) => m.id !== id); + // manifest 摘除后,页面映射转入"退休表":在途读者仍可按 id 读取 + pageStore?.retirePageIds(id); + await this.commitManifest(); + }, + deleteManyMetas: async (ids) => { + if (ids.length === 0) return; + const state = nsState(); + const removing = new Set(ids); + state.sstables = state.sstables.filter((m) => !removing.has(m.id)); + for (const id of ids) pageStore?.retirePageIds(id); + await this.commitManifest(); + }, + retire: (ids) => { + for (const id of ids) pageStore?.retirePageIds(id); }, }; } @@ -2135,7 +2590,6 @@ export class AriaEngine implements IStorageEngine { // 清除主 LSM 中该表前缀的所有数据(含 SSTable 中的旧数据) const prefix = `${tableName}:`; const endKey = `${prefix}\uffff`; - await this.lsm.prefetchRange(prefix, endKey); const entries = await this.lsm.rangeScan(prefix, endKey); for (const [key] of entries) { this.lsm.delete(key); @@ -2261,21 +2715,17 @@ export class AriaEngine implements IStorageEngine { if (col === pkCol) { if (typeof condition !== 'object' || condition === null) { const key = `${tableName}:${condition}`; - await this.lsm.prefetchKeys([key]); const value = await this.lsm.get(key); return value ? [{ ...value, [pkCol]: condition }] : []; } const cond = condition as Record; if ('$eq' in cond) { const key = `${tableName}:${cond.$eq}`; - await this.lsm.prefetchKeys([key]); const value = await this.lsm.get(key); return value ? [{ ...value, [pkCol]: cond.$eq }] : []; } // v0.3.3: PK $in → 主 LSM 多次精确查找(替代冗余 PK 二级索引) if ('$in' in cond && Array.isArray(cond.$in)) { - const keys = cond.$in.map((v) => `${tableName}:${v}`); - await this.lsm.prefetchKeys(keys); const rows: Record[] = []; const seen = new Set(); // v0.4.1: IN 子查询可能含重复值,按 pk 去重 for (const v of cond.$in) { @@ -2289,7 +2739,6 @@ export class AriaEngine implements IStorageEngine { // v0.3.3: PK 范围查询 → 主 LSM 前缀扫描 + 条件过滤(修复字符串算术 bug) if ('$gt' in cond || '$gte' in cond || '$lt' in cond || '$lte' in cond) { const prefix = `${tableName}:`; - await this.lsm.prefetchRange(prefix, `${prefix}\uffff`); const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`); const rows: Record[] = []; for (const [key, value] of entries) { @@ -2336,7 +2785,6 @@ export class AriaEngine implements IStorageEngine { // 后台 compaction 长耗时时 N 倍放大(与 v0.6.1 修的 insert 批量预加载 // 性能悬崖同类)。批级预加载后循环内同步 rangeScan/get。 const values = c.$in.map((v) => String(v)); - await idxLsm.prefetchPrefixRanges(values.map((v): [string, string] => [v, `${v}\uffff`])); const results: Record[] = []; const seenPks = new Set(); // v0.4.1: IN 值可能重复,按 pk 去重 const pks: string[] = []; @@ -2350,7 +2798,6 @@ export class AriaEngine implements IStorageEngine { } } } - await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`)); for (const pk of pks) { const row = await this.lsm.get(`${tableName}:${pk}`); if (row) results.push({ ...row, [pkCol]: pk }); @@ -2379,14 +2826,12 @@ export class AriaEngine implements IStorageEngine { // 使用前缀扫描:endKey 需要包含 \uffff 以匹配所有带后缀的 key const actualEndKey = endKey.includes('\uffff') ? endKey : `${endKey}\uffff`; // 预加载索引 LSM 与主 LSM 涉及的 SSTable - await idxLsm.prefetchRange(startKey, actualEndKey); const entries = await idxLsm.rangeScan(startKey, actualEndKey); const pks: string[] = []; for (const [, idxEntry] of entries) { const pk = (idxEntry as any).pk as string; if (pk) pks.push(pk); } - await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`)); const rows: Record[] = []; for (const pk of pks) { const row = await this.lsm.get(`${tableName}:${pk}`); @@ -2522,18 +2967,28 @@ export class AriaEngine implements IStorageEngine { */ async vacuum(): Promise<{ compactedLevels: number; gcVersions: number }> { this.ensureOpen(); - // 强制 flush memtable + // 强制 flush memtable(vacuum 的语义是"把可回收的空间收掉",必须先把内存数据落盘) await this.lsm.flush(); - // 压缩各层级 - for (let level = 0; level < 6; level++) { - if (this.lsm.getStats().levelCounts[level] >= 2) { - await this.lsm.compactLevel(level); - } + // v0.8.0(B-6):压缩**真实发生过的**层,并返回真实计数。 + // + // 修复前:循环 `level < 6`(写死,MAX_LSM_LEVELS = 7,**底部层永远不压缩** + // → 墓碑与历史版本在最底层永久累积),且无论是否真的合并过都返回 + // `compactedLevels: 6`("报告的数字与事实无关",审计 item 52)。 + // 现在逐层尝试(含底部层的原地合并 —— 它会回收墓碑),只统计真正合并了的层。 + let compactedLevels = 0; + const isBottom = (level: number): boolean => level === MAX_LSM_LEVELS - 1; + for (let level = 0; level < MAX_LSM_LEVELS; level++) { + const files = this.lsm.getStats().levelCounts[level] ?? 0; + // 底部层即使只有 1 个文件也要合并:那正是"墓碑/历史版本回收"的唯一时机 + //(删除密集场景下底部层通常就是一个大文件) + const minFiles = isBottom(level) ? 1 : 2; + if (files < minFiles) continue; + if (await this.lsm.compactLevel(level, minFiles)) compactedLevels++; } // GC MVCC 版本(保留最新 10 个) const beforeGC = this.mvcc.getGlobalLSN(); this.mvcc.gc(10); - return { compactedLevels: 6, gcVersions: beforeGC }; + return { compactedLevels, gcVersions: beforeGC }; } private ensureOpen(): void { diff --git a/src/engine/aria/index/lsm.ts b/src/engine/aria/index/lsm.ts index 1d1d7d2..734e4d0 100644 --- a/src/engine/aria/index/lsm.ts +++ b/src/engine/aria/index/lsm.ts @@ -6,6 +6,25 @@ * * v0.2.1: 完整持久化 — SSTable 元数据和数据均存入存储后端, * 启动时自动扫描并加载所有 SSTable。 + * + * v0.8.0(B-6)本层根治的四类结构缺陷: + * 1. **冻结表没有重试路径**(审计 44):后台 flush 失败后数据只存在于内存, + * 而 `flush()` 又因"后台错误检查在入链之前"直接抛错跳过本次刷盘 → 崩溃即丢。 + * 现在冻结表是一等状态(`FrozenTable`),失败后保留并可由后续 `flush()` + * 重新入链;错误在**完成入链之后**才报告。 + * 2. **compaction 整层摘除**(审计 50):`splice()` 先摘掉整层再去加载/合并, + * 长 await 窗口内该层对读者不可见(静默少行)。现在旧 meta 在合并提交前 + * 一直留在 `levels` 中,提交点之后再原子替换。 + * 3. **在途读者与物理删除竞态**:读路径是"取 meta 快照 → 加载文件", + * 中间可插入 compaction。现在用**读者 epoch** 延迟物理删除: + * 只要有更早进入的读者仍在进行,被取代的文件就不删。 + * 4. **墓碑/历史版本永不回收**(审计 51):底部层新增原地合并(drop tombstones), + * 删除密集场景的空间不再无界增长;同时 `compacting` 由单 boolean 改为 + * **按层集合**(审计 49),跨层触发不再被静默丢弃。 + * + * 另外把"读路径必须先 prefetch"这条隐式约定彻底删除(审计 55 的另一半): + * 读取自洽(未命中即回源 + CRC 校验)后,引擎层的 `drainChain()+prefetch*` + * 全部消失,checkpoint 也不必再等完整 compaction。 */ import { MemTable } from './memtable'; @@ -14,6 +33,7 @@ import { SSTableReader } from './sstable'; import { MergeIterator, ArrayEntrySource, GeneratorEntrySource } from './merge_iterator'; import { DatabaseError } from '../../../constants'; import type { SSTableMeta } from '../types'; +import type { ManifestFrozenIntent } from '../store/manifest'; import { DEFAULT_MEMTABLE_SIZE, MAX_LSM_LEVELS, @@ -34,18 +54,26 @@ export interface SSTableStore { * 若仍写未压缩长度,压缩数据会被 0 填充撑大(静默损坏)。 */ save(id: number, data: Uint8Array): Promise<{ storedSize: number }>; - /** 加载 SSTable 文件 */ + /** 加载 SSTable 文件(返回 null = 文件确实不存在;**介质故障必须抛错**) */ load(id: number): Promise; - /** 删除 SSTable 文件 */ + /** 删除 SSTable 文件(物理删除) */ delete(id: number): Promise; /** 分配下一个 SSTable ID */ allocateId(): Promise; - /** 列出所有已存储的 SSTable 元数据 */ + /** 列出所有已存储的 SSTable 元数据(权威来源:manifest) */ listMeta(): Promise; - /** 保存 SSTable 元数据 */ + /** 保存 SSTable 元数据(= manifest 提交点) */ saveMeta(meta: SSTableMeta): Promise; - /** 删除 SSTable 元数据 */ + /** 删除 SSTable 元数据(= manifest 提交点) */ deleteMeta(id: number): Promise; + /** v0.8.0(可选):批量摘除元数据(一次 manifest 提交;compaction 退休用) */ + deleteManyMetas?(ids: number[]): Promise; + /** + * v0.8.0(可选):把 SSTable 标记为"已退休"——元数据已从 manifest 摘除, + * 但在途读者的快照仍可能按 id 读取它。实现方需保留按 id 定位数据的能力 + * (页面化存储:把 pageIds 移入退休表),直到物理 `delete()` 被调用。 + */ + retire?(ids: number[]): void; } // --------------------------------------------------------------------------- @@ -60,6 +88,93 @@ export interface LSMConfig { sstableStore: SSTableStore; /** SSTable 缓存容量上限(字节),超限按 LRU 驱逐 */ cacheLimitBytes?: number; + /** v0.8.0:命名空间名(manifest 冻结表意图与诊断用),默认 'main' */ + namespace?: string; + /** + * v0.8.0:WAL LSN 提供者。冻结 memtable 时调用,把"该表内容全部来自这之后的 + * 记录"这一事实写进 manifest 的冻结表意图 —— 它是 WAL 水位不得越过的下限, + * 也是恢复期"已确认写入是否真的还在"的校验依据。未提供时按 0 处理。 + */ + walLsnProvider?: () => number; + /** + * v0.8.0:是否要求"被 manifest 引用的 SSTable 必须完好"。 + * + * 由引擎依据 `manifest.wal.startLsn > 0` 传入:WAL 已被截断的情况下, + * SSTable 缺失/损坏就意味着**已确认的数据真的没了** —— 此时必须在恢复报告里 + * 标记 `dataLossSuspected`(而不是像修复前那样只打印一条 warn 就继续)。 + */ + requireDurableCoverage?: boolean; +} + +/** 单次 flush 的最大尝试轮数(每轮把所有未落盘冻结表重新入链一次) */ +const MAX_FLUSH_RETRY_ROUNDS = 3; + +/** 触发 compaction 的文件数门槛(自动调度) */ +const COMPACT_TRIGGER_FILES = 4; +/** 单层积压过多时的背压门槛 */ +const BACKPRESSURE_FILES = 8; + +/** + * 读路径"结构版本一致"的最大重试次数(v0.8.0)。 + * 超过则退回到"等后台链静默"的保守读取(见 collectReaders)。 + */ +const MAX_READ_STRUCTURE_RETRIES = 16; + +// --------------------------------------------------------------------------- +// 冻结表(pending flush 的一等状态,v0.8.0) +// --------------------------------------------------------------------------- + +interface FrozenTable { + /** 本 LSM 内单调递增的冻结表 id(manifest 意图用) */ + id: number; + memtable: MemTable; + /** 冻结时刻的 WAL LSN(manifest 记录水位下限) */ + lsnAtFreeze: number; + /** 是否已在链上排队(防止同一张表被重复入链 → 重复 SSTable) */ + queued: boolean; + /** + * v0.8.0:该表正在被"提交为 SSTable"(`saveMeta` 已发出、manifest 提交进行中)。 + * + * 为什么需要:manifest 的 `frozen` 意图是"这批数据还没落盘"的声明。而 + * `saveMeta` 正是在**提交这批数据本身** —— 此刻把它继续写进 frozen 列表, + * 会让恢复误判"数据没落盘"(实测:从上一代 manifest 回退时抛出 + * ARIA_WRITE_LOST,而数据其实完好)。提交中的表由这一次 manifest 提交负责, + * 因此从意图列表中排除;提交失败时标志复位、表仍在列表里(可重试)。 + */ + committing: boolean; + /** 已尝试次数 */ + attempts: number; + /** 最近一次失败原因 */ + lastError: unknown; +} + +/** SSTable 恢复诊断 */ +export interface LSMRecoveryReport { + namespace: string; + /** 打开/读取过程中被丢弃的 SSTable */ + droppedSSTables: { id: number; level: number; reason: string }[]; + /** 是否怀疑已确认数据丢失(被丢弃的 SSTable 没有 WAL 兜底) */ + dataLossSuspected: boolean; +} + +export interface LSMStats { + memtableSize: number; + sstableCount: number; + levelCounts: number[]; + /** v0.8.0:待落盘冻结表数量(失败后仍保留,可重试) */ + frozenTables: number; + /** v0.8.0:退休但仍有在途读者、尚未物理删除的 SSTable 数量 */ + retiredTables: number; + /** v0.8.0:正在 compaction 的层 */ + compactingLevels: number[]; + /** + * v0.8.0:因"前台/后台结构在读取期间发生变化"而重试的读取次数(累计)。 + * + * 这是**可观测的**:数值长期偏高说明写入/维护压力大(读路径在反复重取快照), + * 同时也是回归测试判断"结构版本校验真的生效"的抓手 —— 没有它,重试只能靠 + * "结果对不对"间接推断,而某些场景下不重试也能得到正确结果(纯属运气)。 + */ + readStructureRetries: number; } // --------------------------------------------------------------------------- @@ -68,9 +183,10 @@ export interface LSMConfig { export class LSM { private memtable: MemTable; - private immutableMemtable: MemTable | null = null; + private immutableMemtable: FrozenTable | null = null; /** 所有 pending flush 的 frozen memtable(含 immutableMemtable,旧→新) */ - private frozenMemtables: MemTable[] = []; + private frozenMemtables: FrozenTable[] = []; + private frozenIdCounter = 0; private levels: SSTableMeta[][] = []; private sstableCache: Map = new Map(); private cacheSize = 0; @@ -84,14 +200,53 @@ export class LSM { private sstableStore: SSTableStore; private operationCount = 0; private initialized = false; - private compacting = false; // 防止重复触发 compaction - /** 串行化 flush/compaction 链:保证持久化顺序与 id 分配顺序一致 */ + /** v0.8.0(审计 49):按层的 compaction 进行集合(此前单 boolean 会静默丢弃跨层触发) */ + private compacting = new Set(); + /** 串行化 memtable flush 链:保证持久化顺序与 id 分配顺序一致 */ private flushChain: Promise = Promise.resolve(); + /** + * v0.8.0(审计 55):compaction 独立于 flush 的维护链。 + * + * 修复前二者共用一条链,于是"每 1000 次操作触发的 checkpoint"必须等 + * 完整 compaction(含级联)跑完 —— 这就是 v0.6.1 记录的"8~11s 悬崖"。 + * 现在 flush 只等自己的任务;只有 close/repair 这类需要完全静默的路径 + * 才等待维护链。 + */ + private maintenanceChain: Promise = Promise.resolve(); /** * v0.4.3-fix: 最近一次后台 flush/compaction 失败。 * 后台失败不卡死链(吞错防死锁),但在显式 flush()/close() 时报告(不静默)。 */ private lastBackgroundError: unknown = null; + /** v0.8.0:已被重试修复的后台故障(不阻塞 flush,但要可见) */ + private backgroundWarnings: unknown[] = []; + /** v0.8.0:读者 epoch —— 用于延迟物理删除被 compaction 取代的文件 */ + private readEpoch = 0; + /** v0.8.0:读路径结构版本重试计数(诊断 + 回归断言用) */ + private readStructureRetries = 0; + private activeReaders = new Map(); + /** v0.8.0:已从 levels 摘除、等待"没有更早读者"后再物理删除的 SSTable */ + private retired: { epoch: number; metas: SSTableMeta[] }[] = []; + /** + * v0.8.0:`levels` 结构版本 —— 每次有 SSTable 发布/摘除就 +1。 + * + * 为什么必须有它:读路径要"取 meta 快照 → await 加载文件",而 flush 可以在 + * 任意 await 点把新 SSTable 发布到 `levels[0]` **并同时**把它从 + * `frozenMemtables` 摘掉。此时本次读既没有在快照里看到新 SSTable、 + * 又不再能从前台冻结表读到那批数据 —— **刚写入的行在这一次扫描里凭空消失** + *(实测:缓存驱逐场景下 300 行全部改名,紧接着的全表扫描有 25 行仍是旧值; + * 几毫秒后同一 key 又能读出新值)。 + * + * 旧实现靠"读之前先 drainChain + prefetch"回避了这个窗口(代价是每次读都要 + * 等后台链)。现在改为**乐观重试**:加载完读取器后若结构版本变了就重来; + * 极端情况下(写入持续不断)才退回到"等 flush/维护链静默"的保守路径。 + */ + private structureVersion = 0; + /** v0.8.0:恢复诊断 */ + private recoveryReport: LSMRecoveryReport; + private readonly namespace: string; + private readonly walLsnProvider: () => number; + private readonly requireDurableCoverage: boolean; constructor(config: LSMConfig) { this.memtableSizeThreshold = config.memtableSizeThreshold ?? DEFAULT_MEMTABLE_SIZE; @@ -100,6 +255,14 @@ export class LSM { this.blockSize = config.blockSize ?? 4096; this.sstableStore = config.sstableStore; this.cacheLimitBytes = config.cacheLimitBytes ?? 64 * 1024 * 1024; + this.namespace = config.namespace ?? 'main'; + this.walLsnProvider = config.walLsnProvider ?? (() => 0); + this.requireDurableCoverage = config.requireDurableCoverage ?? false; + this.recoveryReport = { + namespace: this.namespace, + droppedSSTables: [], + dataLossSuspected: false, + }; for (let i = 0; i < MAX_LSM_LEVELS; i++) { this.levels.push([]); @@ -116,7 +279,9 @@ export class LSM { const metas = await this.sstableStore.listMeta(); // v0.4.2-fix: 打开时完整性校验 — 验证每个 meta 引用的文件存在、可解析, - // 残缺/损坏的 SSTable 忽略并清理 meta,避免后续读取抛 RangeError 崩溃 + // 残缺/损坏的 SSTable 忽略并清理 meta,避免后续读取抛 RangeError 崩溃。 + // v0.8.0(B-6):清理动作进恢复报告;WAL 已被截断(requireDurableCoverage)时 + // 标记 dataLossSuspected —— 数据真的没了,必须让上层看得见。 const validMetas: SSTableMeta[] = []; for (const meta of metas) { if (await this.validateSSTable(meta)) { @@ -137,12 +302,10 @@ export class LSM { this.levels[i].sort((a, b) => b.id - a.id); } - // 注意:不在此处预加载全部 SSTable 数据。 - // 缓存容量有上限(cacheLimitBytes),全部预加载会突破内存预算。 - // 读取路径由 prefetchRange / prefetchKeys 在查询前异步加载兜底。 - // id 序列由 sstableStore.allocateId() 按命名空间独立恢复。 - + // v0.8.0:此时已不再需要"先 prefetch 再读"的隐式约定 —— + // 读取自洽(缓存未命中即回源),因此打开时不预加载任何 SSTable。 this.initialized = true; + this.structureVersion++; } // ======================================================================= @@ -151,7 +314,7 @@ export class LSM { put(key: string, value: Record): void { // 写背压:level 0 SSTable 过多时排队 compaction 缓解压力 - if (this.levels[0].length >= 8) { + if (this.levels[0].length >= BACKPRESSURE_FILES) { this.enqueueCompact(0); } this.memtable.put(key, value); @@ -162,7 +325,7 @@ export class LSM { } delete(key: string): void { - if (this.levels[0].length >= 8) { + if (this.levels[0].length >= BACKPRESSURE_FILES) { this.enqueueCompact(0); } this.memtable.put(key, { __tombstone: true } as unknown as Record); @@ -196,7 +359,7 @@ export class LSM { /** 获取估算内存使用(字节) */ getEstimatedMemory(): number { let mem = this.memtable.getEstimatedSize(); - for (const frozen of this.frozenMemtables) mem += frozen.getEstimatedSize(); + for (const frozen of this.frozenMemtables) mem += frozen.memtable.getEstimatedSize(); mem += this.cacheSize; return mem; } @@ -211,13 +374,50 @@ export class LSM { */ freezeMemtable(): void { if (this.immutableMemtable) { - const frozen = this.immutableMemtable; - this.flushChain = this.enqueueOnChain(() => this.flushImmutableAsync(frozen)); + this.enqueueFlush(this.immutableMemtable); } - this.immutableMemtable = this.memtable; - this.frozenMemtables.push(this.immutableMemtable); + const frozen: FrozenTable = { + id: ++this.frozenIdCounter, + memtable: this.memtable, + lsnAtFreeze: this.walLsnProvider(), + queued: false, + committing: false, + attempts: 0, + lastError: null, + }; + this.immutableMemtable = frozen; + this.frozenMemtables.push(frozen); // v0.4.2-fix: 新 memtable 用配置阈值(此前传旧表已用大小 → 阈值逐次衰减 → 频繁小文件 flush) this.memtable = new MemTable(this.memtableSizeThreshold); + // v0.8.0:**前台变化也算结构变化**。读路径的"结构版本一致"必须覆盖 + // memtable/frozen 这一侧:冻结会把数据从活跃 memtable 挪到 frozen 列表, + // 若版本号不变,在途读者(已经做过前台检查、正在加载 SSTable)会认为 + // "什么都没变"→ 既不重试、又读不到刚挪走的数据 → 返回旧值/漏行。 + this.structureVersion++; + } + + /** + * v0.8.0:把一张冻结表入链(幂等 —— 已在链上的表不会重复入链)。 + * + * 失败语义:任务失败时把错误记在冻结表上(`lastError`)并继续抛给链的 + * catch(记录 `lastBackgroundError`),**冻结表本身保留** —— 于是下一次 + * `flush()` 可以把它重新入链重试(修复前失败即永久失去落盘机会)。 + */ + private enqueueFlush(frozen: FrozenTable): void { + if (frozen.queued) return; + frozen.queued = true; + this.flushChain = this.enqueueOnChain(async () => { + frozen.attempts++; + try { + await this.flushImmutableAsync(frozen); + frozen.lastError = null; + } catch (error) { + frozen.lastError = error; + throw error; + } finally { + frozen.queued = false; + } + }); } /** v0.4.2-fix: 在串行链上排队任务;任务失败吞错并记录,保证链不被单次失败卡死 */ @@ -233,6 +433,17 @@ export class LSM { }); } + /** v0.8.0:维护(compaction)链的入链(失败同样不卡死链) */ + private enqueueMaintenance(task: () => Promise): void { + this.maintenanceChain = this.maintenanceChain + .then(task) + .catch((error) => { + this.lastBackgroundError = error; + // eslint-disable-next-line no-console + console.warn('[AriaEngine LSM] background compaction failed:', error); + }); + } + /** * v0.4.3-fix: 排空后台链 — 循环等待 flushChain 直到稳定。 * 任务完成时可能级联调度新任务(compaction 多级触发),单次 await 等不到。 @@ -242,15 +453,25 @@ export class LSM { while (true) { const chain = this.flushChain; await chain; - if (this.flushChain === chain) return; + if (this.flushChain === chain) break; } } - /** 将指定 Immutable MemTable 刷盘为 SSTable(id 由 store 按命名空间分配) */ - private async flushImmutableAsync(frozen: MemTable): Promise { - const entries = frozen.getAllEntries(); + /** v0.8.0:排空维护链(compaction) */ + private async drainMaintenance(): Promise { + while (true) { + const chain = this.maintenanceChain; + await chain; + if (this.maintenanceChain === chain) break; + } + } + + /** 将指定冻结表刷盘为 SSTable(id 由 store 按命名空间分配) */ + private async flushImmutableAsync(frozen: FrozenTable): Promise { + const entries = frozen.memtable.getAllEntries(); if (entries.length === 0) { if (this.immutableMemtable === frozen) this.immutableMemtable = null; + this.removeFrozen(frozen); return; } @@ -272,64 +493,103 @@ export class LSM { bloomData: null, }; - // 缓存(缓存的是内存中的**未压缩**整文件,与存储布局无关) + // 持久化:先存数据(落盘完成才返回),再提交元数据(manifest 提交点) + // v0.8.0:缓存放到落盘成功**之后** —— 修复前先缓存再落盘,失败重试会留下 + // 永不使用却占内存的缓存条目。 + const stored = await this.sstableStore.save(id, sstableData); + if (!stored || typeof stored.storedSize !== 'number') { + throw new DatabaseError( + `SSTableStore.save() returned ${JSON.stringify(stored)} for namespace "${this.namespace}" ` + + `(id=${id}, bytes=${sstableData.byteLength}) — 存储实现违反了 save() 契约`, + 'ARIA_SSTABLE_SAVE_CONTRACT', + ); + } + meta.totalSize = stored.storedSize; + // 提交窗口:本表的数据由这次 manifest 提交负责 → 不再出现在 frozen 意图里 + frozen.committing = true; + try { + await this.sstableStore.saveMeta(meta); + } finally { + frozen.committing = false; + } + + // 提交成功后才在内存中可见 → 读路径不会看到"元数据已提交但文件未落盘" + this.levels[0].unshift(meta); + this.structureVersion++; this.tryCacheSSTable(id, sstableData); this.trimCache(); - - // 持久化:先存数据,再存元数据(串行链保证顺序与 id 一致) - const stored = await this.sstableStore.save(id, sstableData); - meta.totalSize = stored.storedSize; - await this.sstableStore.saveMeta(meta); - if (this.immutableMemtable === frozen) this.immutableMemtable = null; - this.levels[0].unshift(meta); - // 数据已落盘(levels 可见),从 pending frozen 移除 - this.frozenMemtables = this.frozenMemtables.filter((f) => f !== frozen); + this.removeFrozen(frozen); // 异步触发 compaction(不阻塞当前写入) - if (this.levels[0].length >= 4 && !this.compacting) { + if (this.levels[0].length >= COMPACT_TRIGGER_FILES) { this.scheduleCompact(0); } } + /** + * 从 pending 列表移除冻结表。 + * + * 这里**不**单独 bump 结构版本:调用方只有两处 —— + * ① 落盘成功(同一步里 `levels[0].unshift(meta)` 已经 bump); + * ② 空表直接丢弃(没有任何数据,读者观察不到差异)。 + * 多余的版本号增长只会让读路径白白重试。 + */ + private removeFrozen(frozen: FrozenTable): void { + this.frozenMemtables = this.frozenMemtables.filter((f) => f !== frozen); + } + + // ======================================================================= + // Compaction + // ======================================================================= + + /** + * 执行 Compaction(public,供 VACUUM 等外部调用;默认 2 个文件即可压缩)。 + * + * @returns 是否**真的发生了一次合并**(VACUUM 用它统计真实压缩层数; + * 修复前 VACUUM 无论是否合并都硬编码报告 6 层) + */ + async compactLevel(level: number, minFiles: number = 2): Promise { + // VACUUM 是显式维护操作:直接等待(与后台调度区分) + return this.compactLevelAsync(level, minFiles); + } + /** * 异步调度 compaction。 * v0.4.3-fix: 去掉 setTimeout 分片 — 此前未触发的定时器在 close 后执行, * 用已关闭的 backend 写存储(错误被吞),或 close 后 reopen 时旧闭包引用新 backend 交叉污染。 - * 现在直接挂在 flushChain 上:串行执行、close 的 drainChain 能等到全部完成。 + * 现在直接挂在维护链上:串行执行、close 的 drainMaintenance 能等到全部完成。 */ private scheduleCompact(level: number): void { - if (level >= MAX_LSM_LEVELS - 1 || this.compacting) return; - this.compacting = true; - this.flushChain = this.enqueueOnChain(() => this.compactLevelAsync(level).finally(() => { - this.compacting = false; - // 连续触发:如果 compaction 后仍然超标,继续调度 - if (this.levels[level].length >= 4) { - this.scheduleCompact(level); + if (level > MAX_LSM_LEVELS - 1 || this.compacting.has(level)) return; + this.compacting.add(level); + this.enqueueMaintenance(async () => { + try { + await this.compactLevelAsync(level); + } finally { + this.compacting.delete(level); + // 连续触发:如果 compaction 后仍然超标,继续调度 + if (this.levels[level].length >= COMPACT_TRIGGER_FILES) { + this.scheduleCompact(level); + } + // 检查下一级是否需要 compaction(底部层的"原地合并"同样允许) + if (level + 1 <= MAX_LSM_LEVELS - 1 && this.levels[level + 1].length >= COMPACT_TRIGGER_FILES) { + this.scheduleCompact(level + 1); + } } - // 检查下一级是否需要 compaction - if (level + 1 < MAX_LSM_LEVELS - 1 && this.levels[level + 1].length >= 4) { - this.scheduleCompact(level + 1); - } - })); + }); } /** 背压场景下排队 compaction(写入路径调用) */ private enqueueCompact(level: number): void { - if (level >= MAX_LSM_LEVELS - 1 || this.compacting) return; - this.compacting = true; - this.flushChain = this.flushChain.then(async () => { + if (level > MAX_LSM_LEVELS - 1 || this.compacting.has(level)) return; + this.compacting.add(level); + this.enqueueMaintenance(async () => { try { await this.compactLevelAsync(level); } finally { - this.compacting = false; + this.compacting.delete(level); } - }).catch((error) => { - this.compacting = false; - // v0.4.3-fix: 记录失败(flush()/close() 时报告) - this.lastBackgroundError = error; - // eslint-disable-next-line no-console - console.warn('[AriaEngine LSM] background compaction failed:', error); }); } @@ -337,127 +597,188 @@ export class LSM { // 读取 // ======================================================================= - /** - * 预加载指定 key 范围内可能命中的所有 SSTable 到缓存。 - * 在同步扫描/查找之前调用,保证 loadSSTableReader 不会因缓存未命中而返回 null。 - * v0.4.3-fix: 等待链稳定(drainChain)— 后台 flush/compaction 在链上动态增长, - * 单次 await 后 compaction 仍可能合并 levels,导致扫描时新 meta 缓存未命中而跳块丢数据。 + /** v0.8.0:登记一次读操作(epoch),返回 epoch */ + private enterRead(): number { + const epoch = ++this.readEpoch; + this.activeReaders.set(epoch, (this.activeReaders.get(epoch) ?? 0) + 1); + return epoch; + } + + /** v0.8.0:读操作结束;顺带回收已无在途读者的退休文件 */ + private exitRead(epoch: number): void { + const count = this.activeReaders.get(epoch); + if (count === undefined) return; + if (count <= 1) this.activeReaders.delete(epoch); + else this.activeReaders.set(epoch, count - 1); + this.reclaimRetired(); + } + + /** v0.8.0:在**没有任何 await** 的同步片段里取 meta 快照。 + * + * 必须同步取快照的原因:读路径后面要 `await` 加载 SSTable,而 compaction + * 可以在任意 await 点改变 `levels`。若边遍历边 await,遍历会看到半更新的 + * 层数组(漏文件/重复文件)。快照 + epoch 保护把这件事变成显式的: + * 快照期间的层内容由 `retired` 机制保证文件不会被删。 + * + * 注意:快照本身不足以保证"完整"(并发 flush 可能在加载期间发布新 SSTable), + * 因此调用方必须配合 `structureVersion` 做乐观重试 —— 见 `collectReaders`。 */ - async prefetchRange(startKey: string, endKey: string): Promise { - await this.drainChain(); - this.trimCache(); - const toLoad: number[] = []; + private snapshotMetas(startKey: string, endKey: string): SSTableMeta[] { + const out: SSTableMeta[] = []; for (let level = 0; level < MAX_LSM_LEVELS; level++) { for (const meta of this.levels[level]) { if (endKey < meta.minKey || startKey > meta.maxKey) continue; - if (!this.sstableCache.has(meta.id)) toLoad.push(meta.id); + out.push(meta); } } - for (const id of toLoad) { - const meta = this.findMetaById(id); - await this.preloadSSTable(id, meta); - } - } - - /** 预加载包含指定 key 的所有 SSTable 到缓存 */ - async prefetchKeys(keys: string[]): Promise { - if (keys.length === 0) return; - // v0.4.3-fix: 等待链稳定(同 prefetchRange,防 compaction 竞态丢数据) - await this.drainChain(); - this.trimCache(); - const toLoad = new Set(); - for (let level = 0; level < MAX_LSM_LEVELS; level++) { - for (const meta of this.levels[level]) { - if (this.sstableCache.has(meta.id)) continue; - for (const key of keys) { - if (key >= meta.minKey && key <= meta.maxKey) { - toLoad.add(meta.id); - break; - } - } - } - } - for (const id of toLoad) { - const meta = this.findMetaById(id); - await this.preloadSSTable(id, meta); - } + return out; } /** - * v0.6.2: 批量预加载多个前缀范围可能命中的所有 SSTable(一次 drainChain)。 - * 唯一性检查等"每行一个窄前缀范围"场景用:meta 遍历只做一次, - * 避免逐行调用 prefetchRange 时每行 drainChain 的性能悬崖。 + * v0.8.0:取一个**自洽**的读取器集合(含并发 flush/compaction 的正确性)。 + * + * 算法:快照 → 加载 → 若期间结构变过则重来。重试只由"确有新产物发布"触发, + * 因此正常情况一次成功;写入极端密集时退回到"等 flush/维护链静默"的保守路径 + * (等价于旧实现的行为,但只在极少数情况下付出这个代价)。 */ - async prefetchPrefixRanges(ranges: [string, string][]): Promise { - if (ranges.length === 0) return; - // v0.6.2: 去重(大批量插入时唯一列值可能重复),减少 meta 遍历开销 - const seen = new Set(); - const unique: [string, string][] = []; - for (const r of ranges) { - const k = `${r[0]}\u0000${r[1]}`; - if (!seen.has(k)) { - seen.add(k); - unique.push(r); + private async collectReaders(startKey: string, endKey: string): Promise { + for (let attempt = 0; attempt < MAX_READ_STRUCTURE_RETRIES; attempt++) { + const version = this.structureVersion; + const metas = this.snapshotMetas(startKey, endKey); + const readers: SSTableReader[] = []; + for (const meta of metas) { + const reader = await this.loadSSTableReader(meta); + if (reader) readers.push(reader); } + if (version === this.structureVersion) return readers; + this.readStructureRetries++; } - if (unique.length === 0) return; + // 保守回退:等两条链静默后再取一次(此时不可能再有结构变化) await this.drainChain(); - this.trimCache(); - const toLoad = new Set(); - for (let level = 0; level < MAX_LSM_LEVELS; level++) { - for (const meta of this.levels[level]) { - if (this.sstableCache.has(meta.id)) continue; - for (const [startKey, endKey] of unique) { - if (endKey < meta.minKey || startKey > meta.maxKey) continue; - toLoad.add(meta.id); - break; - } - } - } - for (const id of toLoad) { - const meta = this.findMetaById(id); - await this.preloadSSTable(id, meta); + await this.drainMaintenance(); + const readers: SSTableReader[] = []; + for (const meta of this.snapshotMetas(startKey, endKey)) { + const reader = await this.loadSSTableReader(meta); + if (reader) readers.push(reader); } + return readers; } - /** 按 id 查找 SSTable 元数据(prefetch 预加载用) */ - private findMetaById(id: number): SSTableMeta | undefined { - for (let level = 0; level < MAX_LSM_LEVELS; level++) { - for (const meta of this.levels[level]) { - if (meta.id === id) return meta; - } + /** v0.8.0:回收退休 SSTable —— 只有当"可能持有它们的读者"全部退出后才物理删除 */ + private reclaimRetired(): void { + if (this.retired.length === 0) return; + let minActive = Infinity; + for (const epoch of this.activeReaders.keys()) { + if (epoch < minActive) minActive = epoch; } - return undefined; + const keep: { epoch: number; metas: SSTableMeta[] }[] = []; + const toDelete: SSTableMeta[] = []; + for (const group of this.retired) { + // 该组在 epoch=group.epoch 时被摘除:所有更早进入的读者都可能持有它 + if (group.epoch < minActive) toDelete.push(...group.metas); + else keep.push(group); + } + this.retired = keep; + if (toDelete.length === 0) return; + void (async () => { + for (const meta of toDelete) { + this.sstableCache.delete(meta.id); + this.oversizedSSTables.delete(meta.id); + try { + await this.sstableStore.delete(meta.id); + } catch (error) { + // 物理删除失败只造成空间残留(repair 可回收),不影响正确性 + // eslint-disable-next-line no-console + console.warn(`[AriaEngine LSM] retired SSTable id=${meta.id} deletion failed:`, error); + } + } + })(); + } + + /** v0.8.0(测试/诊断):退休但尚未物理删除的 SSTable 数量 */ + getRetiredCount(): number { + return this.retired.reduce((sum, g) => sum + g.metas.length, 0); + } + + /** + * v0.8.0:**强制**回收全部退休 SSTable(不等更早的读者退出)。 + * + * 只允许在"确定没有在途读者"的维护路径上调用(`repair()` 会先排空两条链)。 + * 语义:把这些文件当作已无引用的垃圾删除,并清空退休登记。 + */ + reclaimRetiredNow(): void { + if (this.retired.length === 0) return; + const all = this.retired.flatMap((g) => g.metas); + this.retired = []; + void (async () => { + for (const meta of all) { + this.sstableCache.delete(meta.id); + this.oversizedSSTables.delete(meta.id); + try { + await this.sstableStore.delete(meta.id); + } catch { /* 残留由 repair 的孤儿回收处理 */ } + } + })(); } /** * v0.8.0: 改为 async —— SSTable 部分未命中缓存时会 `await` 回源, * 因此读取不再依赖调用方的 prefetch(消除"缓存未命中即静默丢数据")。 + * + * v0.8.0:整体包在"结构版本一致"的重试里(见 collectReaders)—— + * 前台(memtable/frozen)与后台(SSTable)两侧必须在同一个结构版本下读, + * 否则并发 flush 会让刚写的数据在这一刻既不在前台也不在快照里。 */ async get(key: string): Promise | null> { - // 1. 活跃 MemTable - let result = this.memtable.get(key); - if (result !== null) return this.unwrapTombstone(result); - - // 2. pending frozen memtables(从新到旧) - for (let i = this.frozenMemtables.length - 1; i >= 0; i--) { - result = this.frozenMemtables[i].get(key); + for (let attempt = 0; attempt < MAX_READ_STRUCTURE_RETRIES; attempt++) { + // 1. 活跃 MemTable + let result = this.memtable.get(key); if (result !== null) return this.unwrapTombstone(result); - } - // 3. SSTable(从 Level 0 到 Level N-1) - for (let level = 0; level < MAX_LSM_LEVELS; level++) { - for (const meta of this.levels[level]) { - if (key < meta.minKey || key > meta.maxKey) continue; + // 2. pending frozen memtables(从新到旧) + for (let i = this.frozenMemtables.length - 1; i >= 0; i--) { + result = this.frozenMemtables[i].memtable.get(key); + if (result !== null) return this.unwrapTombstone(result); + } - const reader = await this.loadSSTableReader(meta); - if (!reader) continue; - - const found = reader.get(key); - if (found !== null) return this.unwrapTombstone(found); + // 3. SSTable(从 Level 0 到 Level N-1)—— 快照 + 加载 + 版本校验 + const version = this.structureVersion; + const epoch = this.enterRead(); + try { + const metas = this.snapshotMetas(key, key); + const readers: SSTableReader[] = []; + for (const meta of metas) { + const reader = await this.loadSSTableReader(meta); + if (reader) readers.push(reader); + } + if (version !== this.structureVersion) { + this.readStructureRetries++; + continue; // 期间有结构变化(新产物发布 / 前台冻结)→ 重来 + } + for (const reader of readers) { + const found = reader.get(key); + if (found !== null) return this.unwrapTombstone(found); + } + return null; + } finally { + this.exitRead(epoch); } } - + // 保守回退:等两条链静默后按稳定结构读一次 + await this.drainChain(); + await this.drainMaintenance(); + let result = this.memtable.get(key); + if (result !== null) return this.unwrapTombstone(result); + for (let i = this.frozenMemtables.length - 1; i >= 0; i--) { + result = this.frozenMemtables[i].memtable.get(key); + if (result !== null) return this.unwrapTombstone(result); + } + for (const meta of this.snapshotMetas(key, key)) { + const reader = await this.loadSSTableReader(meta); + if (!reader) continue; + const found = reader.get(key); + if (found !== null) return this.unwrapTombstone(found); + } return null; } @@ -471,64 +792,58 @@ export class LSM { * 惰性范围扫描:通过回调逐条返回,不一次性物化。 * v0.7.4: 真惰性 —— 各源(MemTable/frozen/SSTable)以生成器接入 MergeIterator, * 逐条拉取;回调返回 false 时提前终止(未消费部分不再解析/物化)。 - * 此前实现内部 mergeIter.drain() 全量物化,与"流式不物化"宣称不符。 + * v0.8.0: 快照式读者 epoch —— 扫描期间被 compaction 取代的文件保持可读, + * 因此"扫到一半 compaction 完成"不会让结果少行。 */ async rangeScanLazy( startKey: string, endKey: string, callback: (key: string, value: Record) => boolean | void, ): Promise { - // v0.8.0: 先把范围内需要的 SSTable 读取器全部取齐(未命中即回源), - // 再做纯内存的归并扫描 —— 使读取自洽,不依赖调用方 prefetch。 - const readers: { meta: SSTableMeta; reader: SSTableReader }[] = []; - for (let level = 0; level < MAX_LSM_LEVELS; level++) { - for (const meta of this.levels[level]) { - if (endKey < meta.minKey || startKey > meta.maxKey) continue; - const reader = await this.loadSSTableReader(meta); - if (reader) readers.push({ meta, reader }); - } - } + const epoch = this.enterRead(); + try { + // 「前台与后台必须来自同一个结构版本」这条不变量由 `collectReaders` 负责 + //(它内部做快照 → 加载 → 版本校验 → 重试)。这里**不再重复校验一次**: + // 重复的检查在语义上是冗余的(从 collectReaders 返回到源添加之间没有 await, + // 结构不可能变化),而"看起来需要但实际不生效"的代码只会误导后来者。 + const readers = await this.collectReaders(startKey, endKey); - const mergeIter = new MergeIterator(); - - mergeIter.addSource(new GeneratorEntrySource( - this.memtable.scanLazy(startKey, endKey), - )); - - // pending frozen memtables(从新到旧,新数据 sourceIndex 更小) - for (let i = this.frozenMemtables.length - 1; i >= 0; i--) { + // 从这里到源添加完成**不得有 await**(否则前台列表可能在中间变化) + const mergeIter = new MergeIterator(); mergeIter.addSource(new GeneratorEntrySource( - this.frozenMemtables[i].scanLazy(startKey, endKey), + this.memtable.scanLazy(startKey, endKey), )); - } - - for (const { reader } of readers) { - mergeIter.addSource(new GeneratorEntrySource( - reader.scanLazy(startKey, endKey), - )); - } - - let entry = mergeIter.next(); - while (entry) { - const [k, v] = entry; - if (!(v as unknown as Record).__tombstone) { - const cont = callback(k, v); - // v0.7.4: 提前终止(流式 limit 达成) - if (cont === false) return; + // pending frozen memtables(从新到旧,新数据 sourceIndex 更小) + for (let i = this.frozenMemtables.length - 1; i >= 0; i--) { + mergeIter.addSource(new GeneratorEntrySource( + this.frozenMemtables[i].memtable.scanLazy(startKey, endKey), + )); } - entry = mergeIter.next(); + for (const reader of readers) { + mergeIter.addSource(new GeneratorEntrySource( + reader.scanLazy(startKey, endKey), + )); + } + + let entry = mergeIter.next(); + while (entry) { + const [k, v] = entry; + if (!(v as unknown as Record).__tombstone) { + const cont = callback(k, v); + // v0.7.4: 提前终止(流式 limit 达成) + if (cont === false) return; + } + entry = mergeIter.next(); + } + } finally { + this.exitRead(epoch); } } // ======================================================================= - // Compaction + // Compaction 实现 // ======================================================================= - /** 执行 Compaction(public,供 VACUUM 等外部调用;VACUUM 期望 2 个文件即可压缩) */ - async compactLevel(level: number): Promise { - await this.compactLevelAsync(level, 2); - } - /** * 串行执行 Compaction。 * @param minFiles 触发压缩的文件数门槛(自动调度用 4,VACUUM 用 2) @@ -536,38 +851,56 @@ export class LSM { * v0.4.2-fix: 读取从存储兜底(不依赖缓存)——此前仅从缓存读, * 缓存未命中(LRU 驱逐/单文件超缓存上限)时跳过全部文件并从 levels 移除, * 运行中数据全部不可见。 + * + * v0.8.0(审计 50/51)两处结构改动: + * - **不再 `splice` 整层**:合并期间旧 meta 留在 `levels`(读者始终看得到完整数据), + * 合并提交后才按 id 原子摘除,同窗口内新 flush 进来的产物不受影响; + * - **底部层原地合并 + 回收墓碑**:`level === MAX_LSM_LEVELS - 1` 时输出回同层, + * 并丢弃墓碑(底部层没有更老的数据,丢墓碑不会复活已删除行)。 */ - private async compactLevelAsync(level: number, minFiles: number = 4): Promise { - if (level >= MAX_LSM_LEVELS - 1) return; - if (this.levels[level].length < minFiles) return; + private async compactLevelAsync(level: number, minFiles: number = COMPACT_TRIGGER_FILES): Promise { + if (level > MAX_LSM_LEVELS - 1) return false; + if (this.levels[level].length < minFiles) return false; - const sstables = this.levels[level].splice(0, this.levels[level].length); + const isBottomLevel = level === MAX_LSM_LEVELS - 1; + const targetLevel = isBottomLevel ? level : level + 1; + + // 选择源文件:**整层**合并(保持"层内键范围完整"这一不变量)。 + // 用快照 + 提交后按 id 摘除,而不是先 splice 再慢慢加载。 + const selected = [...this.levels[level]]; + const selectedIds = new Set(selected.map((m) => m.id)); const mergeIter = new MergeIterator(); const loadedMetas: SSTableMeta[] = []; - for (const meta of sstables) { + for (const meta of selected) { // 优先缓存,未命中则从存储加载(残缺文件经校验清理,跳过) let data: Uint8Array | null = this.sstableCache.get(meta.id) ?? null; if (!data) { try { data = await this.sstableStore.load(meta.id); - } catch { - data = null; + } catch (error) { + // v0.8.0:介质读故障 != 文件不存在。宁可让 compaction 失败(可重试), + // 也不能把读故障当成"文件损坏"从而丢弃整层数据。 + throw new DatabaseError( + `AriaEngine compaction failed to read SSTable id=${meta.id}: ${(error as Error).message}`, + 'ARIA_SSTABLE_READ_FAILED', + error, + ); } } if (!data || data.byteLength < 32) { - await this.dropInvalidSSTable(meta); + await this.dropInvalidSSTable(meta, 'missing or truncated during compaction'); continue; } let reader: SSTableReader; try { reader = new SSTableReader(data, meta); if (!reader.verifyChecksum()) { - await this.dropInvalidSSTable(meta); + await this.dropInvalidSSTable(meta, 'checksum mismatch during compaction'); continue; } - } catch { - await this.dropInvalidSSTable(meta); + } catch (error) { + await this.dropInvalidSSTable(meta, `unparseable during compaction: ${(error as Error).message}`); continue; } const entries: [string, Record][] = []; @@ -576,15 +909,28 @@ export class LSM { loadedMetas.push(meta); } - const merged = mergeIter.drain(); + const mergedRaw = mergeIter.drain(); + // v0.8.0(审计 51):底部层丢弃墓碑(该 key 在底部层之外没有更老的数据) + const merged = isBottomLevel + ? mergedRaw.filter(([, value]) => !(value as unknown as Record).__tombstone) + : mergedRaw; if (merged.length === 0) { - // 没有有效数据(全部损坏):把有效 meta 放回 levels, - // 避免文件从读取路径消失(数据仍在磁盘,重启可恢复) - for (const meta of loadedMetas) { - this.levels[level].push(meta); + if (mergedRaw.length === 0) { + // 没有有效数据(全部损坏):把有效 meta 放回 levels, + // 避免文件从读取路径消失(数据仍在磁盘,重启可恢复) + for (const meta of loadedMetas) { + if (!this.levels[level].some((m) => m.id === meta.id)) this.levels[level].push(meta); + } + this.levels[level].sort((a, b) => b.id - a.id); + this.structureVersion++; + return false; } - this.levels[level].sort((a, b) => b.id - a.id); - return; + // 底部层合并后只剩墓碑:整层都是已删除行 → 产物为空。 + // 这是"删空整张表后再合并"的必然路径:必须按**退休**处理(摘除 meta + + // 延迟物理删除),而不是继续走"写入空 SSTable"(merged[0] 不存在 → + // 会抛 TypeError,compaction 永远失败、墓碑永远回收不掉)。 + await this.retireMetas(level, this.removeFromLevel(level, selectedIds)); + return true; } const id = await this.sstableStore.allocateId(); @@ -596,7 +942,7 @@ export class LSM { const { sstableData, indexEntries } = builder.build(); const meta: SSTableMeta = { id, - level: level + 1, + level: targetLevel, minKey: merged[0][0], maxKey: merged[merged.length - 1][0], blockCount: indexEntries.length, @@ -605,67 +951,238 @@ export class LSM { bloomData: null, }; - this.tryCacheSSTable(id, sstableData); - this.trimCache(); const stored = await this.sstableStore.save(id, sstableData); - meta.totalSize = stored.storedSize; - await this.sstableStore.saveMeta(meta); - this.levels[level + 1].unshift(meta); - - // 删除旧 SSTable - for (const old of sstables) { - this.sstableCache.delete(old.id); - await this.sstableStore.delete(old.id); - await this.sstableStore.deleteMeta(old.id); - } - } - - /** 等待所有排队的 flush/compaction 完成,并将剩余数据刷盘 */ - async flush(): Promise { - // v0.4.3-fix: 报告后台失败(消费一次,不永久吞错) - if (this.lastBackgroundError !== null) { - const error = this.lastBackgroundError; - this.lastBackgroundError = null; + if (!stored || typeof stored.storedSize !== 'number') { throw new DatabaseError( - 'AriaEngine background flush/compaction failed (data may be inconsistent)', - 'ARIA_BACKGROUND_ERROR', - error, + `SSTableStore.save() returned ${JSON.stringify(stored)} for namespace "${this.namespace}" ` + + `(id=${id}, bytes=${sstableData.byteLength}) — 存储实现违反了 save() 契约`, + 'ARIA_SSTABLE_SAVE_CONTRACT', ); } - // v0.4.3-fix: 循环等待级联任务(flush 完成可能触发新的 compaction) + meta.totalSize = stored.storedSize; + // 提交点 1:先提交合并产物(崩溃在摘除旧 meta 之前 → manifest 同时含新旧 meta, + // 二者内容等价/新者更新,读路径仍然正确) + await this.sstableStore.saveMeta(meta); + + this.tryCacheSSTable(id, sstableData); + this.trimCache(); + this.levels[targetLevel].unshift(meta); + this.structureVersion++; + + // 提交点 2:摘除被取代的旧 meta(一次提交)。摘除后仍留在 retired 中—— + // 在途读者还能读到它们的数据,直到所有更早的读者退出才物理删除。 + await this.retireMetas(level, this.removeFromLevel(level, selectedIds)); + return true; + } + + /** + * v0.8.0:从某一层**实际**摘除给定 id 的 meta(同步),返回被摘除的那些。 + * + * 只摘除"此刻确实还在这一层"的条目:合并窗口内可能有并发 compaction/clear + * 已经处理过同一批文件,重复退休会让物理删除做两次(无害但会掩盖真实状态)。 + */ + private removeFromLevel(level: number, ids: Set): SSTableMeta[] { + const removed: SSTableMeta[] = []; + this.levels[level] = this.levels[level].filter((m) => { + if (ids.has(m.id)) { + removed.push(m); + return false; + } + return true; + }); + return removed; + } + + /** + * v0.8.0:把被取代的 SSTable 从某一层摘除并进入"退休"状态。 + * + * 顺序固定为:先从 levels 摘除(同步)→ 登记 retired(延迟物理删除)→ + * 再由 manifest 一次提交摘除 meta。三个动作共同保证: + * - 在途读者仍能读到旧数据(文件在 retired 里活着); + * - 崩溃在摘除 meta 之前时 manifest 同时含新旧 meta(内容等价/新者更新); + * - 崩溃在摘除之后时,新的合并产物已提交(数据完整)。 + */ + private async retireMetas(level: number, metas: SSTableMeta[]): Promise { + if (metas.length === 0) return; + this.structureVersion++; + this.retired.push({ epoch: this.readEpoch, metas }); + this.sstableStore.retire?.(metas.map((m) => m.id)); + const ids = metas.map((m) => m.id); + if (typeof this.sstableStore.deleteManyMetas === 'function') { + await this.sstableStore.deleteManyMetas(ids); + } else { + for (const oldId of ids) await this.sstableStore.deleteMeta(oldId); + } + // 摘除动作完成后再尝试回收(正常情况下还有在途读者,会推迟到读者退出时) + this.reclaimRetired(); + } + + /** + * v0.8.0(B-6/44+45):把所有 pending 数据刷盘。 + * + * 与修复前的区别(这两个行为此前都是缺陷): + * 1. **先把当前 memtable 入链,再报告后台错误** —— 修复前错误检查在入链之前, + * 于是一次后台失败会让本次 flush 完全不执行(数据继续只存在于内存里); + * 2. **失败可重试** —— 修复前失败的冻结表虽然还留在 `frozenMemtables` 里可读, + * 却没有任何重新入链的机会(唯一入链点是 freeze/flush 当时那一次), + * 下次 `flush()` 又因第 1 点直接抛错,冻结数据永远等不到落盘。 + * + * 返回时的保证:**所有已确认写入(put/delete 已返回)的数据都在已提交的 + * SSTable 中**;做不到则抛 `ARIA_BACKGROUND_ERROR`(不静默)。 + */ + async flush(): Promise { + this.enqueuePendingMemtables(); + const reported = this.consumeBackgroundError(); await this.drainChain(); - // v0.6.1-fix(P0): 剩余数据 flush 必须挂链串行 —— 此前直接 await 执行, - // 与链上 compaction 并发写 SSTable meta:compaction 产物 saveMeta 后, - // memtable flush 的 saveMeta 读到中间态列表(含 compaction 产物)→ 覆盖产物 - // 引用 → compaction 产物变孤儿 → 索引/主表数据静默丢失(优雅关闭后重开丢 75%)。 - // 挂链后按序执行(memtable flush 在 compaction 之后),meta 无竞态。 - // - // v0.6.3-fix: ① 入链后立即置空 immutable —— 此前 freezeMemtable 会把同一张 - // immutable 再次入链 → 重复 SSTable(2 行 flush 出 3 个文件);② frozenMemtables - // 保持到链排空后再清理 —— 此前入链后立即清空,链执行期间并发读看不到冻结数据 - // (短暂数据不可见窗口)。 + await this.retryPendingFlushes(); + // close/repair 语义:等维护链(compaction)也静默下来 + await this.drainMaintenance(); + this.assertNoPendingFrozen(reported); + } + + /** + * v0.8.0(B-6/55):只把 memtable 落盘,**不等 compaction**。 + * + * checkpoint 需要的是"WAL 覆盖的数据已落盘",而 compaction 只是重排已经 + * 落盘的数据 —— 因此 checkpoint 不必(也不应)等它。这就是 v0.6.1 记录的 + * "8~11s 悬崖"的根治:写路径的周期 checkpoint 不再被 compaction 拖住。 + */ + async flushMemtablesOnly(): Promise { + this.enqueuePendingMemtables(); + const reported = this.consumeBackgroundError(); + await this.drainChain(); + await this.retryPendingFlushes(); + this.assertNoPendingFrozen(reported); + } + + /** 把当前 memtable(若有数据)入链 */ + private enqueuePendingMemtables(): void { if (this.immutableMemtable) { const frozen = this.immutableMemtable; - this.flushChain = this.enqueueOnChain(() => this.flushImmutableAsync(frozen)); this.immutableMemtable = null; + this.enqueueFlush(frozen); } if (this.memtable.getEntryCount() > 0) { this.freezeMemtable(); - const frozen = this.immutableMemtable; - if (frozen) { - this.flushChain = this.enqueueOnChain(() => this.flushImmutableAsync(frozen)); + if (this.immutableMemtable) { + const frozen = this.immutableMemtable; this.immutableMemtable = null; + this.enqueueFlush(frozen); } } - // 刷盘完成后级联调度可能触发 compaction → 排空到稳定 - await this.drainChain(); - // 清理空残留冻结表(flushImmutableAsync 完成时已自行移除已落盘的) - this.frozenMemtables = this.frozenMemtables.filter((f) => f.getEntryCount() > 0); + } + + /** 失败的冻结表重新入链(有界重试) */ + private async retryPendingFlushes(): Promise { + let rounds = 0; + while (this.frozenMemtables.some((f) => !f.queued && f.memtable.getEntryCount() > 0)) { + if (rounds >= MAX_FLUSH_RETRY_ROUNDS) break; + rounds++; + for (const frozen of this.frozenMemtables) { + if (frozen.memtable.getEntryCount() > 0) this.enqueueFlush(frozen); + } + await this.drainChain(); + } + } + + /** 消费一次后台错误(返回后 `lastBackgroundError` 已清空) */ + private consumeBackgroundError(): unknown { + if (this.lastBackgroundError === null) return null; + const error = this.lastBackgroundError; + this.lastBackgroundError = null; + return error; + } + + /** flush 结束后的断言:不得还有未落盘的冻结表 */ + private assertNoPendingFrozen(reported: unknown): void { + // flush 过程中新产生的后台错误同样算"本次 flush 的错误"(不能在下次 flush + // 才冒出来打断一次本来完全成功的调用) + const late = this.consumeBackgroundError(); + const effective = reported ?? late; + const pending = this.frozenMemtables.filter((f) => f.memtable.getEntryCount() > 0); + if (pending.length > 0) { + const lastError = pending[pending.length - 1].lastError; + throw new DatabaseError( + `AriaEngine LSM flush could not persist ${pending.length} frozen memtable(s) after ` + + `${MAX_FLUSH_RETRY_ROUNDS} retry round(s) — data is still only in memory (WAL is the only other copy)`, + 'ARIA_BACKGROUND_ERROR', + lastError ?? effective, + ); + } + if (effective !== null && effective !== undefined) { + // 失败已被重试修复:不阻塞 flush(数据确实已落盘),但必须可见。 + // + // 为什么不在下一次 flush 抛错:那会让"上一次的瞬时故障"随机打断一次 + // 本来完全成功的 flush(调用方会据此回滚已经落盘的数据)。可见性由 + // `getBackgroundWarnings()` 与告警日志提供,错误语义只保留 + // "数据没落盘才算失败"。 + this.backgroundWarnings.push(effective); + // eslint-disable-next-line no-console + console.warn('[AriaEngine LSM] background failure recovered by retry:', effective); + } + } + + /** v0.8.0:被重试修复的后台故障(诊断/测试) */ + getBackgroundWarnings(): unknown[] { + return [...this.backgroundWarnings]; + } + + /** v0.8.0:是否还有未落盘的 memtable 数据(manifest 水位不得越过它) */ + hasPendingFlushData(): boolean { + if (this.memtable.getEntryCount() > 0) return true; + return this.frozenMemtables.some((f) => f.memtable.getEntryCount() > 0); + } + + /** + * v0.8.0:待落盘冻结表意图(写入 manifest)。 + * `lsnAtFreeze` 是 WAL 水位下限 —— 它保证"只存在于内存的冻结数据"不会被 + * 当成已落盘而截断掉。 + */ + getFrozenIntents(): ManifestFrozenIntent[] { + const intents: ManifestFrozenIntent[] = []; + for (const frozen of this.frozenMemtables) { + // 正在提交的表由当前这次 manifest 提交负责(见 FrozenTable.committing) + if (frozen.committing) continue; + const entries = frozen.memtable.getAllEntries(); + if (entries.length === 0) continue; + intents.push({ + ns: this.namespace, + id: frozen.id, + entryCount: entries.length, + minKey: entries[0][0], + maxKey: entries[entries.length - 1][0], + lsnAtFreeze: frozen.lsnAtFreeze, + }); + } + if (this.immutableMemtable && !this.immutableMemtable.committing) { + const entries = this.immutableMemtable.memtable.getAllEntries(); + if (entries.length > 0 && !intents.some((i) => i.id === this.immutableMemtable!.id)) { + intents.push({ + ns: this.namespace, + id: this.immutableMemtable.id, + entryCount: entries.length, + minKey: entries[0][0], + maxKey: entries[entries.length - 1][0], + lsnAtFreeze: this.immutableMemtable.lsnAtFreeze, + }); + } + } + return intents; + } + + /** v0.8.0:恢复诊断(被丢弃的 SSTable / 是否怀疑数据丢失) */ + getRecoveryReport(): LSMRecoveryReport { + return { + namespace: this.recoveryReport.namespace, + droppedSSTables: [...this.recoveryReport.droppedSSTables], + dataLossSuspected: this.recoveryReport.dataLossSuspected, + }; } async clear(): Promise { // v0.4.3-fix: 清空前排空后台任务(避免 compaction 在清空后写回残留 meta/数据) await this.drainChain(); + await this.drainMaintenance(); this.memtable.clear(); this.immutableMemtable = null; this.frozenMemtables = []; @@ -685,13 +1202,19 @@ export class LSM { this.sstableCache.clear(); this.cacheSize = 0; this.oversizedSSTables.clear(); + this.retired = []; + this.structureVersion++; } - getStats(): { memtableSize: number; sstableCount: number; levelCounts: number[] } { + getStats(): LSMStats { return { memtableSize: this.memtable.getEntryCount(), sstableCount: this.levels.reduce((sum, l) => sum + l.length, 0), levelCounts: this.levels.map((l) => l.length), + frozenTables: this.frozenMemtables.filter((f) => f.memtable.getEntryCount() > 0).length, + retiredTables: this.getRetiredCount(), + compactingLevels: [...this.compacting].sort((a, b) => a - b), + readStructureRetries: this.readStructureRetries, }; } @@ -704,20 +1227,26 @@ export class LSM { * @returns 移除的损坏 SSTable 数量 */ async validateAll(): Promise { - let removed = 0; - for (let level = 0; level < MAX_LSM_LEVELS; level++) { - const valid: SSTableMeta[] = []; - for (const meta of this.levels[level]) { - if (await this.validateSSTable(meta)) { - valid.push(meta); - } else { - this.sstableCache.delete(meta.id); - removed++; + const epoch = this.enterRead(); + try { + let removed = 0; + for (let level = 0; level < MAX_LSM_LEVELS; level++) { + const valid: SSTableMeta[] = []; + for (const meta of this.levels[level]) { + if (await this.validateSSTable(meta)) { + valid.push(meta); + } else { + this.sstableCache.delete(meta.id); + removed++; + } } + this.levels[level] = valid; } - this.levels[level] = valid; + this.structureVersion++; + return removed; + } finally { + this.exitRead(epoch); } - return removed; } /** @@ -725,40 +1254,45 @@ export class LSM { * - 文件不存在 → 清理 meta,返回 false * - 文件过小/魔数错误/索引越界(残缺写入产物)→ 清理 meta,返回 false * - v0.4.5: 整文件 CRC-32 校验失败(数据腐坏)→ 清理 meta,返回 false - * 校验通过的数据不缓存(保持内存预算),读路径按需预加载。 + * v0.8.0: 介质读故障(抛错)**不**当作"文件不存在",直接向上抛 —— + * 否则一次瞬时读错误就会让整个 SSTable 的 meta 被删掉(不可逆)。 */ private async validateSSTable(meta: SSTableMeta): Promise { - try { - const data = await this.sstableStore.load(meta.id); - if (!data) { - this.dropInvalidSSTable(meta); - return false; - } - if (data.byteLength < 32) { - this.dropInvalidSSTable(meta); - return false; - } - try { - const reader = new SSTableReader(data, meta); - if (!reader.verifyChecksum()) { - this.dropInvalidSSTable(meta); - return false; - } - } catch { - this.dropInvalidSSTable(meta); - return false; - } - return true; - } catch { - this.dropInvalidSSTable(meta); + const data = await this.sstableStore.load(meta.id); + if (!data) { + await this.dropInvalidSSTable(meta, 'file missing at open'); return false; } + if (data.byteLength < 32) { + await this.dropInvalidSSTable(meta, `file too small (${data.byteLength} bytes)`); + return false; + } + try { + const reader = new SSTableReader(data, meta); + if (!reader.verifyChecksum()) { + await this.dropInvalidSSTable(meta, 'checksum mismatch at open'); + return false; + } + } catch (error) { + await this.dropInvalidSSTable(meta, `unparseable at open: ${(error as Error).message}`); + return false; + } + return true; } - /** 清理无效 SSTable 的 meta 与文件(打开自愈路径) */ - private async dropInvalidSSTable(meta: SSTableMeta): Promise { + /** 清理无效 SSTable 的 meta 与文件(打开自愈路径)+ 记录恢复诊断 */ + private async dropInvalidSSTable(meta: SSTableMeta, reason: string): Promise { + this.recoveryReport.droppedSSTables.push({ id: meta.id, level: meta.level, reason }); + if (this.requireDurableCoverage) { + // manifest 已推进 WAL 水位(startLsn > 0)→ 被丢弃的 SSTable 没有 WAL 兜底 + this.recoveryReport.dataLossSuspected = true; + } // eslint-disable-next-line no-console - console.warn(`[AriaEngine LSM] Skipping corrupted SSTable id=${meta.id} (level=${meta.level})`); + console.warn( + `[AriaEngine LSM] Dropping unusable SSTable id=${meta.id} (level=${meta.level}): ${reason}`, + ); + this.sstableCache.delete(meta.id); + this.oversizedSSTables.delete(meta.id); // v0.6.0-fix: 先删数据文件再删 meta — 页面化存储的 delete 依赖 meta.pageIds // 定位页面文件;先删 meta 会丢失 pageIds 导致孤儿页面残留 try { @@ -775,7 +1309,6 @@ export class LSM { return value; } - /** 尝试从缓存或存储加载 SSTable,返回 Reader */ /** * v0.8.0 根治:读取器加载**自洽**(缓存未命中即回源),不再依赖调用方预先 prefetch。 * @@ -786,28 +1319,45 @@ export class LSM { * 小缓存下 300 行只能查回 59 行(且不报错)。同时"读路径必须先 prefetch" * 这个隐式约定,也是每次读都要 drainChain + prefetch 的原因(性能悬崖的另一半)。 * - * 现在:未命中就 `await sstableStore.load()` 回源并校验;只有**确实读不到数据** - * (文件缺失/损坏,已由 dropInvalidSSTable 处理)才返回 null,且会在自愈时清掉 meta。 + * 语义区分(v0.8.0 补齐第三态): + * - 读到了 → 返回 reader; + * - **确定不存在**且仍被 manifest 引用 → 自愈清理该 meta 并进恢复报告; + * - **确定不存在但已被 compaction 摘除**(退休)→ 静默返回 null + * (在途读者的快照必然也已看到了合并产物,只是顺序上还没轮到); + * - **读取抛错(介质故障)** → 向上抛,绝不折叠成"不存在"。 */ private async loadSSTableReader(meta: SSTableMeta): Promise { let data = this.sstableCache.get(meta.id); if (!data) { - const loaded = await this.sstableStore.load(meta.id); + let loaded: Uint8Array | null; + try { + loaded = await this.sstableStore.load(meta.id); + } catch (error) { + throw new DatabaseError( + `AriaEngine failed to read SSTable id=${meta.id} from storage: ${(error as Error).message}`, + 'ARIA_SSTABLE_READ_FAILED', + error, + ); + } if (!loaded) { - // 数据确实不存在:清理该 meta(自愈),避免每次查询都重试 - await this.dropInvalidSSTable(meta); + // 数据确实读不到:区分"仍被引用的缺失"(损坏,需自愈 + 报告) + // 与"已被 compaction 取代"(正常退休,静默跳过) + const referenced = (await this.sstableStore.listMeta()).some((m) => m.id === meta.id); + if (referenced) { + await this.dropInvalidSSTable(meta, 'file missing during read'); + } return null; } // 运行期回源同样校验整文件 CRC-32(与 preloadSSTable 一致) try { const probe = new SSTableReader(loaded, meta); if (!probe.verifyChecksum()) { - await this.dropInvalidSSTable(meta); + await this.dropInvalidSSTable(meta, 'checksum mismatch during read'); return null; } - } catch { - await this.dropInvalidSSTable(meta); + } catch (error) { + await this.dropInvalidSSTable(meta, `unparseable during read: ${(error as Error).message}`); return null; } this.tryCacheSSTable(meta.id, loaded); @@ -828,27 +1378,6 @@ export class LSM { } } - /** 预加载 SSTable 到缓存(受 cacheLimitBytes 上限约束) */ - async preloadSSTable(id: number, meta?: SSTableMeta): Promise { - if (this.sstableCache.has(id)) return; - const data = await this.sstableStore.load(id); - if (!data) return; - // v0.4.5: 运行期加载同样校验整文件 CRC-32,损坏文件不缓存并清理(自愈) - if (meta) { - try { - const reader = new SSTableReader(data, meta); - if (!reader.verifyChecksum()) { - await this.dropInvalidSSTable(meta); - return; - } - } catch { - await this.dropInvalidSSTable(meta); - return; - } - } - this.tryCacheSSTable(id, data); - } - /** * v0.8.0: 单文件大于缓存上限时的处理 —— **不缓存**,且不牵连其他条目。 * @@ -863,11 +1392,8 @@ export class LSM { private tryCacheSSTable(id: number, data: Uint8Array): void { if (data.byteLength > this.cacheLimitBytes) { // 单个文件就超过整个缓存上限:**必须常驻**。 - // 若把它驱逐,`loadSSTableReader` 缓存未命中会返回 null,而调用方的 - // `if (!reader) continue` 会**静默跳过该文件** —— 查询结果直接少数据 - // (实测 300 行只返回 59 行)。这也是审计指出的"缓存未命中与不存在不可区分"。 - // 因此这类文件标记为 pinned,不参与驱逐;代价是内存占用可超出 cacheLimit, - // 上限为 cacheLimit + 单个最大 SSTable。 + // 若把它驱逐,`loadSSTableReader` 缓存未命中会回源(正确但慢), + // 而 pinned 语义让热点大文件避免反复解码。 this.oversizedSSTables.add(id); } else { this.oversizedSSTables.delete(id); @@ -892,8 +1418,8 @@ export class LSM { * 查询结束后由引擎调用一次,回收查询期间的临时超限。 */ trimCache(): void { - // v0.8.0: 超过缓存上限的单个文件被 pin 住(驱逐它们会导致读取路径静默丢数据), - // 因此这里的循环只驱逐未 pin 的条目;若只剩 pinned 条目则接受超限。 + // v0.8.0: 超过缓存上限的单个文件被 pin 住(它们只是不参与 LRU, + // 读取路径已自洽:驱逐后回源即可),因此这里只驱逐未 pin 的条目。 let scanned = 0; const total = this.sstableCache.size; let id = this.sstableCache.keys().next().value as number | undefined; diff --git a/src/engine/aria/index/merge_iterator.ts b/src/engine/aria/index/merge_iterator.ts index 45dd9f4..62a71ef 100644 --- a/src/engine/aria/index/merge_iterator.ts +++ b/src/engine/aria/index/merge_iterator.ts @@ -132,6 +132,19 @@ class MinHeap { export class MergeIterator { private sources: EntrySource[]; private heap: MinHeap; + /** + * v0.8.0(B-6/47):上一次返回的条目所属来源 —— 它的下一条**推迟到下次 next()** + * 才拉取。 + * + * 修复前是"弹出堆顶后立刻补充该来源的下一条",于是**消费者只取 N 条, + * 底层生成器却已经产出 N+1 条**(审计实测:limit=5 的流式扫描多算 1 条)。 + * 在真惰性扫描里这不是纯性能问题:多拉的那一条会解析一整个 SSTable 块, + * 也让"提前终止时未消费部分不再解析"的宣称不完全成立。 + * + * 延迟补充是安全的:单个来源内部 key 唯一且有序,因此被弹出条目的后续 + * key 必然大于当前 key,不可能参与本次的重复 key 归并。 + */ + private pendingRefill: number | null = null; constructor() { this.sources = []; @@ -146,16 +159,20 @@ export class MergeIterator { /** 获取下一个归并后的条目 */ next(): [string, Record] | null { + // 上一轮被延迟的补充:现在才真正拉取(见 pendingRefill 说明) + if (this.pendingRefill !== null) { + const sourceIndex = this.pendingRefill; + this.pendingRefill = null; + this.seedFromSource(sourceIndex); + } if (this.heap.size === 0) return null; const first = this.heap.pop()!; const key = first.key; let best = first; - // 刷新 first 来源的下一个值 - this.seedFromSource(first.sourceIndex); - - // 跳过重复 key:在多个来源中保留 sourceIndex 最小(最新)的条目 + // 跳过重复 key:在多个来源中保留 sourceIndex 最小(最新)的条目。 + // 重复条目必须**立即**从各自来源补充(否则它们会永久占住堆顶)。 while (this.heap.peek() && this.heap.peek()!.key === key) { const dup = this.heap.pop()!; this.seedFromSource(dup.sourceIndex); @@ -164,6 +181,9 @@ export class MergeIterator { } } + // 胜出来源的补充推迟到下一次 next()(消费者只取 N 条 → 底层只产出 N 条) + this.pendingRefill = first.sourceIndex; + return [best.key, best.value]; } diff --git a/src/engine/aria/index/sstable.ts b/src/engine/aria/index/sstable.ts index 8a85985..6bd2144 100644 --- a/src/engine/aria/index/sstable.ts +++ b/src/engine/aria/index/sstable.ts @@ -66,39 +66,9 @@ export class SSTableReader { const blockIdx = this.locateBlock(targetKey); if (blockIdx < 0) return null; - const entry = this.indexEntries[blockIdx]; - const blockData = this.getBlockData(entry); - // v0.4.1-fix: 残缺文件(meta 偏移超出实际长度)跳过该块,而非抛 RangeError - if (!blockData) return null; - const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength); - const lenSize = this.lenFieldSize(); - - const entryCount = blockView.getUint32(0, false); - let offset = 4; - - // 顺序扫描 block 内的条目(生产中应二分查找) - for (let i = 0; i < entryCount; i++) { - if (offset + lenSize > blockData.byteLength) break; - const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); - offset += lenSize; - if (offset + keyLen + lenSize > blockData.byteLength) break; - const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen)); - offset += keyLen; - const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); - offset += lenSize; - if (offset + valLen > blockData.byteLength) break; - const valBytes = blockData.slice(offset, offset + valLen); - offset += valLen; - - if (key === targetKey) { - try { - return JSON.parse(new TextDecoder().decode(valBytes)); - } catch { - return null; - } - } + for (const [key, value] of this.iterEntries(blockIdx, blockIdx)) { + if (key === targetKey) return value; } - return null; } @@ -131,48 +101,49 @@ export class SSTableReader { const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1); if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx) return; - const lenSize = this.lenFieldSize(); - for (let bi = startBlockIdx; bi <= endBlockIdx && bi >= 0; bi++) { - const entry = this.indexEntries[bi]; - const blockData = this.getBlockData(entry); - // v0.4.1-fix: 残缺块跳过(继续后续块,不抛异常) - if (!blockData) continue; - const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength); - - const blockEntryCount = blockView.getUint32(0, false); - let offset = 4; - - for (let i = 0; i < blockEntryCount; i++) { - if (offset + lenSize > blockData.byteLength) break; - const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); - offset += lenSize; - if (offset + keyLen + lenSize > blockData.byteLength) break; - const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen)); - offset += keyLen; - const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); - offset += lenSize; - if (offset + valLen > blockData.byteLength) break; - const valBytes = blockData.slice(offset, offset + valLen); - offset += valLen; - - if (key >= startKey && key <= endKey) { - try { - const value = JSON.parse(new TextDecoder().decode(valBytes)); - yield [key, value]; - } catch { - // skip corrupted entry - } - } - } + for (const [key, value] of this.iterEntries(startBlockIdx, endBlockIdx)) { + if (key >= startKey && key <= endKey) yield [key, value]; } } /** 扫描所有条目 */ scanAll(callback: (key: string, value: Record) => void): void { + for (const [key, value] of this.iterEntries(0, this.indexEntries.length - 1)) { + callback(key, value); + } + } + + // ----------------------------------------------------------------------- + // 统一解析(v0.8.0) + // ----------------------------------------------------------------------- + + /** + * v0.8.0(B-6):**唯一一份**块内条目解析实现。 + * + * 修复前这段解析被抄成三份(`get` / `scanLazy` / `scanAll`),并且三处的 + * 越界策略不一致:点查与扫描对同一个损坏文件可能给出不同结论 + *(审计:`sstable.ts:80-100/145-166/182-201`)。三份实现里只要有一处漏改, + * 就会重新出现"同一文件在不同路径下读出不同数据"。 + * + * 现在的统一策略(对三个调用点完全一致): + * - 块缺失/越界 → 跳过该块,继续后续块(不抛异常); + * - 块内任一条目的长度字段越界 → 该块**剩余条目整体放弃**(截断块),继续后续块; + * - 条目的 JSON 解析失败 → 跳过该条目(视为不存在),不中断其他条目。 + * + * @param fromBlock 起始块下标(含) + * @param toBlock 结束块下标(含) + */ + private *iterEntries( + fromBlock: number, + toBlock: number, + ): Generator<[string, Record]> { const lenSize = this.lenFieldSize(); - for (const entry of this.indexEntries) { - const blockData = this.getBlockData(entry); - // v0.4.1-fix: 残缺块跳过(scanAll 继续后续块,不抛异常) + const start = Math.max(0, fromBlock); + const end = Math.min(toBlock, this.indexEntries.length - 1); + const decoder = new TextDecoder(); + + for (let bi = start; bi <= end; bi++) { + const blockData = this.getBlockData(this.indexEntries[bi]); if (!blockData) continue; const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength); @@ -184,7 +155,7 @@ export class SSTableReader { const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); offset += lenSize; if (offset + keyLen + lenSize > blockData.byteLength) break; - const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen)); + const key = decoder.decode(blockData.slice(offset, offset + keyLen)); offset += keyLen; const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); offset += lenSize; @@ -193,10 +164,10 @@ export class SSTableReader { offset += valLen; try { - const value = JSON.parse(new TextDecoder().decode(valBytes)); - callback(key, value); + const value = JSON.parse(decoder.decode(valBytes)) as Record; + yield [key, value]; } catch { - // skip corrupted entry + // 损坏条目跳过(与三处调用点此前的"跳过损坏条目"策略一致) } } } diff --git a/src/engine/aria/store/file_manager.ts b/src/engine/aria/store/file_manager.ts index 109acdb..3bca4ba 100644 --- a/src/engine/aria/store/file_manager.ts +++ b/src/engine/aria/store/file_manager.ts @@ -24,7 +24,7 @@ export class FileManager implements PageIO { } /** 初始化:从存储中读取元数据 */ - async init(dbName: string): Promise { + async init(dbName: string, watermarkFloor: number = 1): Promise { this.dbName = dbName; const meta = await this.backend.read('__aria_meta'); let nextPageId = 1; @@ -43,11 +43,30 @@ export class FileManager implements PageIO { if (!Number.isNaN(id) && id + 1 > nextPageId) nextPageId = id + 1; } } - this.nextPageId = nextPageId; - if (!meta || !(meta instanceof ArrayBuffer) || meta.byteLength < 4 || nextPageId !== new DataView(meta as ArrayBuffer).getUint32(0, false)) { - await this.saveMeta(); + // v0.8.0(B-6):manifest 的 pageId 水位是**权威下限**(单调推进、永不复用), + // 与"现存最大页面 id + 1"、"旧 __aria_meta" 三者取最大 —— 任何单一来源被 + // 截断/回退都不会导致页面 id 复用。 + if (Number.isFinite(watermarkFloor) && watermarkFloor > nextPageId) { + nextPageId = Math.floor(watermarkFloor); } + this.nextPageId = nextPageId; this.metaLoaded = true; + // v0.8.0(B-6):`__aria_meta` 降级为**兼容/诊断提示**,不再作为提交点: + // 页面水位只由 manifest 提交(单一提交点),这里仅在旧值落后时补写一次, + // 且写入失败不影响引擎(旧版本读到的只是"落后但单调"的提示值)。 + const legacyValue = meta && meta instanceof ArrayBuffer && meta.byteLength >= 4 + ? new DataView(meta as ArrayBuffer).getUint32(0, false) + : -1; + if (legacyValue !== nextPageId) { + try { + await this.writeLegacyWatermark(nextPageId); + } catch { /* 兼容提示写失败不影响正确性 */ } + } + } + + /** v0.8.0: 当前页面 id 水位(下一个可分配 id)—— manifest 提交时记录 */ + getNextPageId(): number { + return this.nextPageId; } // ---- PageIO ---- @@ -78,18 +97,18 @@ export class FileManager implements PageIO { async allocatePageId(): Promise { const id = this.nextPageId++; - await this.saveMeta(); + await this.persistWatermarkHint(); return id; } - /** v0.4.5: 批量分配页面 ID(一次 meta 持久化,避免页面化 SSTable 保存时逐页写 meta) */ + /** v0.4.5: 批量分配页面 ID(一次提示写,避免页面化 SSTable 保存时逐页写 meta) */ async allocatePageIds(count: number): Promise { if (count <= 0) return []; const ids: number[] = []; const start = this.nextPageId; this.nextPageId += count; for (let i = 0; i < count; i++) ids.push(start + i); - await this.saveMeta(); + await this.persistWatermarkHint(); return ids; } @@ -101,9 +120,20 @@ export class FileManager implements PageIO { // ---- 辅助 ---- - private async saveMeta(): Promise { + /** + * v0.8.0(B-6):`__aria_meta` 只是**兼容提示**(旧版本/人工诊断用), + * 失败不抛错 —— 真正的提交点是 manifest 的 `pageIdWatermark`。 + */ + private async persistWatermarkHint(): Promise { + if (!this.metaLoaded) return; + try { + await this.writeLegacyWatermark(this.nextPageId); + } catch { /* 提示写失败不影响正确性(manifest 才是权威) */ } + } + + private async writeLegacyWatermark(nextPageId: number): Promise { const buf = new ArrayBuffer(8); - new DataView(buf).setUint32(0, this.nextPageId, false); + new DataView(buf).setUint32(0, nextPageId, false); await this.backend.write('__aria_meta', buf); } @@ -111,6 +141,8 @@ export class FileManager implements PageIO { async clearAll(): Promise { await this.backend.clear(); this.nextPageId = 1; - await this.saveMeta(); + try { + await this.writeLegacyWatermark(1); + } catch { /* 提示写失败不影响正确性 */ } } } diff --git a/src/engine/aria/store/manifest.ts b/src/engine/aria/store/manifest.ts new file mode 100644 index 0000000..1a07720 --- /dev/null +++ b/src/engine/aria/store/manifest.ts @@ -0,0 +1,723 @@ +/** + * AriaEngine Manifest — 存储层**单一提交点** + * @module engine/aria/store/manifest + * + * v0.8.0(B-6):把原先"四处独立落盘、靠推理保持一致"的元状态收敛成 + * **一份带 CRC 的原子提交记录**: + * + * ``` + * ┌──────────────────────────────────────────────────────────────┐ + * │ 数据落盘(SSTable 页面/文件) │ + * │ ↓ │ + * │ __aria_manifest_ 提交(单文件 COW 原子写 + CRC) │ + * │ ↓ │ + * │ 才允许截断 WAL / 删除旧分片 │ + * └──────────────────────────────────────────────────────────────┘ + * ``` + * + * 修复前的问题(全部是本模块要消除的根因 4 类): + * - `__aria_lsm_meta*` 是**裸 JSON**:`JSON.parse` 失败时 `readMetaList()` 返回 + * `[]` —— 损坏的元数据 = **静默空库**,随后 `repair()` 的孤儿页面清理会 + * 把"没人引用"的活页全部删掉(不可逆); + * - 页面 id 水位(`__aria_meta`)、WAL 起始位置、各命名空间 meta 各自独立落盘, + * 崩溃窗口内三者可以互相矛盾; + * - WAL 截断只看内存状态:**没有任何持久记录**能证明"被截断的记录已落盘"; + * - 陈旧实例(多标签页/多实例)可以直接覆盖新一代的 meta。 + * + * 本模块的语义约定: + * 1. **只认最后一份 CRC 通过的世代**;若存在 manifest 文件但全部世代都无效, + * `load()` **抛错**而不是返回空状态(宁可打不开,也不能静默当空库); + * 2. 提交是**先写后验**:写完立刻回读校验(CRC + 世代号一致)才认为提交成功; + * 3. 至少保留**两代**(当前 + 上一代)用于回退;只有当回退窗口安全时才删除更早的世代; + * 4. 任何"引用不到的东西"在**恢复路径**上一律不删除(删除只发生在显式 repair / + * 已经过提交点确认的 compaction 之后); + * 5. 所有计数器(pageId 水位、各命名空间 SSTable id)**单调推进、永不复用**。 + */ + +import type { ColumnDef } from '../../../constants'; +import { DatabaseError } from '../../../constants'; +import type { SSTableMeta } from '../types'; +import type { IStorageBackend } from './backend'; +import { crc32 } from '../crc32'; + +// --------------------------------------------------------------------------- +// 常量 +// --------------------------------------------------------------------------- + +/** manifest 文件 key 前缀(世代号 8 位十进制) */ +export const MANIFEST_KEY_PREFIX = '__aria_manifest_'; + +/** 魔数 'M' 'S' 'M' 'F'(MetonaSqlark Manifest) */ +export const MANIFEST_MAGIC = 0x4d534d46; + +/** 当前格式版本 */ +export const MANIFEST_FORMAT_VERSION = 1; + +/** 头部字节数:magic(4) + version(2) + headerSize(2) + generation(4) + payloadLen(4) + payloadCrc(4) + headerCrc(4) */ +export const MANIFEST_HEADER_SIZE = 24; + +/** 至少保留的世代数(当前 + 上一代) */ +export const MANIFEST_RETAIN_GENERATIONS = 2; + +/** + * 认领所有权时一次跨过的世代数(v0.8.0)。 + * + * 为什么需要"跨一段"而不是简单地 +1:多实例(浏览器多标签页 / 无 Web Locks 环境) + * 下,旧实例的可能**已经在途**的提交会落在 +1 这个号上,从而覆盖新实例刚认领的 + * 世代 —— 新实例随后的提交就会看到"别人的更高世代"而被判成陈旧实例, + * 两个实例互相拒绝(实测:第二个打开者 open 直接失败)。 + * 一次跨过一段之后,旧实例的在途提交落在更低的号上,既不会覆盖认领, + * 也会在下一次提交时被正常拒绝。 + */ +export const MANIFEST_TAKEOVER_STRIDE = 1000; + +const MANIFEST_KEY_REGEX = /^__aria_manifest_(\d{8})$/; + +// --------------------------------------------------------------------------- +// 类型 +// --------------------------------------------------------------------------- + +/** WAL 的权威位置状态(由 manifest 提交时写入) */ +export interface ManifestWalState { + /** + * 仍需保留的最小分片序号。小于它的分片整体已落盘,可安全删除 + * (删除失败也只是残留文件,恢复时按 `startSegment` 忽略)。 + */ + startSegment: number; + /** + * 落盘水位(LSN 语义):`lsn <= startLsn` 的记录**已确认存在于已提交的 + * SSTable 中**,恢复时可跳过;`startLsn` 只允许在"所有 LSM 均无未落盘数据" + * 的提交点上推进 —— 这是"截断 WAL 前必须先提交 manifest"的可验证形式。 + */ + startLsn: number; + /** LSN 高水位:重启后从此继续递增(LSN 全库单调,永不回退/复用) */ + nextLsn: number; +} + +/** + * 待落盘冻结表意图。 + * + * 冻结表的数据此刻只存在于内存(+ WAL)。把它写进 manifest 有两个作用: + * 1. **阻止水位推进**:`wal.startLsn` 不得越过 `lsnAtFreeze`,否则这些记录会被 + * 当成"已落盘"而跳过(崩溃即永久丢数据); + * 2. **恢复期可校验**:重开后若 manifest 声称有冻结表,但 WAL 里既无对应记录、 + * SSTable 里也无对应数据,说明"已确认写入"真的丢了 —— 此时必须报错, + * 而不是安静地少一批行。 + */ +export interface ManifestFrozenIntent { + /** LSM 命名空间('main' / 'idx__') */ + ns: string; + /** LSM 内部单调递增的冻结表 id */ + id: number; + /** 条目数 */ + entryCount: number; + /** 最小 key */ + minKey: string; + /** 最大 key */ + maxKey: string; + /** 冻结时刻的 WAL LSN(该表内容全部来自这之后的记录) */ + lsnAtFreeze: number; +} + +/** 实例所有权(陈旧实例保护) */ +export interface ManifestOwner { + /** 提交方实例 id */ + instanceId: string; + /** 单调递增的提交世代(每次提交 +1) */ + epoch: number; + /** 首次打开时间(ms) */ + openedAt: number; +} + +/** 单个 LSM 命名空间的持久状态 */ +export interface ManifestNamespaceState { + /** 下一个 SSTable id(单调推进,永不复用) */ + nextSstableId: number; + /** 该命名空间当前全部 SSTable meta(权威列表) */ + sstables: SSTableMeta[]; +} + +/** manifest 完整内容 */ +export interface AriaManifest { + formatVersion: number; + /** 世代号(每次提交 +1,与文件名一致) */ + generation: number; + /** 页面 id 水位(下一页可分配 id;单调推进,永不复用) */ + pageIdWatermark: number; + /** 命名空间 → 状态 */ + namespaces: Record; + /** 表结构(列定义,与旧 `__aria_schemas` 同形) */ + schemas: Record>; + /** WAL 权威位置 */ + wal: ManifestWalState; + /** 待落盘冻结表意图 */ + frozen: ManifestFrozenIntent[]; + /** 所有权 */ + owner: ManifestOwner; + /** 提交时间(ms,诊断用) */ + committedAt: number; +} + +/** 加载结果 */ +export interface ManifestLoadResult { + /** 最新有效世代的内容;null = 全新库(没有任何 manifest 文件) */ + manifest: AriaManifest | null; + /** 最新有效世代号(全新库为 0) */ + generation: number; + /** 是否曾有损坏/不可解析的世代被跳过(保守:存在则禁止自动清理旧世代) */ + hadInvalidGenerations: boolean; + /** 跳过的世代及原因(诊断) */ + skipped: { generation: number; reason: string }[]; +} + +/** 新建空 manifest 的参数 */ +export interface EmptyManifestOptions { + instanceId: string; + now?: number; +} + +// --------------------------------------------------------------------------- +// 空 manifest / 编解码 +// --------------------------------------------------------------------------- + +/** 创建一份空 manifest(全新库;或作为旧格式迁移的起点) */ +export function createEmptyManifest(opts: EmptyManifestOptions): AriaManifest { + return { + formatVersion: MANIFEST_FORMAT_VERSION, + generation: 0, + pageIdWatermark: 1, + namespaces: {}, + schemas: {}, + wal: { startSegment: 0, startLsn: 0, nextLsn: 0 }, + frozen: [], + owner: { instanceId: opts.instanceId, epoch: 0, openedAt: opts.now ?? Date.now() }, + committedAt: opts.now ?? Date.now(), + }; +} + +/** manifest 文件 key */ +export function manifestKey(generation: number): string { + return `${MANIFEST_KEY_PREFIX}${String(generation).padStart(8, '0')}`; +} + +/** 从 key 解析世代号(非 manifest key 返回 null) */ +export function generationFromKey(key: string): number | null { + const m = MANIFEST_KEY_REGEX.exec(key); + return m ? Number(m[1]) : null; +} + +/** 把 manifest 编码为字节(头部 + JSON 载荷) */ +export function encodeManifest(manifest: AriaManifest): Uint8Array { + const payload = new TextEncoder().encode(JSON.stringify(serializeManifest(manifest))); + const buf = new ArrayBuffer(MANIFEST_HEADER_SIZE + payload.byteLength); + const view = new DataView(buf); + view.setUint32(0, MANIFEST_MAGIC, false); + view.setUint16(4, manifest.formatVersion, false); + view.setUint16(6, MANIFEST_HEADER_SIZE, false); + view.setUint32(8, manifest.generation >>> 0, false); + view.setUint32(12, payload.byteLength, false); + view.setUint32(16, payload.byteLength > 0 ? crc32(payload) : 0, false); + // 头部自身也带 CRC(generation/长度被篡改时不会误判为有效世代) + view.setUint32(20, crc32(new Uint8Array(buf, 0, 20)), false); + new Uint8Array(buf, MANIFEST_HEADER_SIZE).set(payload); + return new Uint8Array(buf); +} + +/** 解码结果:成功或明确的失败原因(绝不"失败当空库") */ +export type ManifestDecodeResult = + | { ok: true; manifest: AriaManifest } + | { ok: false; reason: string }; + +/** 从字节解码 manifest(任何异常都转成结构化失败原因) */ +export function decodeManifest(bytes: Uint8Array): ManifestDecodeResult { + try { + if (bytes.byteLength < MANIFEST_HEADER_SIZE) { + return { ok: false, reason: `too small (${bytes.byteLength} < ${MANIFEST_HEADER_SIZE})` }; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const magic = view.getUint32(0, false); + if (magic !== MANIFEST_MAGIC) { + return { ok: false, reason: `bad magic 0x${magic.toString(16)}` }; + } + const headerCrc = view.getUint32(20, false); + const computedHeaderCrc = crc32(bytes.subarray(0, 20)); + if (headerCrc !== computedHeaderCrc) { + return { ok: false, reason: `header CRC mismatch (stored=${headerCrc} computed=${computedHeaderCrc})` }; + } + const version = view.getUint16(4, false); + if (version !== MANIFEST_FORMAT_VERSION) { + return { ok: false, reason: `unsupported format version ${version}` }; + } + const headerSize = view.getUint16(6, false); + if (headerSize !== MANIFEST_HEADER_SIZE) { + return { ok: false, reason: `unexpected header size ${headerSize}` }; + } + const generation = view.getUint32(8, false); + const payloadLength = view.getUint32(12, false); + if (payloadLength === 0 || payloadLength > bytes.byteLength - MANIFEST_HEADER_SIZE) { + return { ok: false, reason: `invalid payload length ${payloadLength}` }; + } + const payload = bytes.subarray(MANIFEST_HEADER_SIZE, MANIFEST_HEADER_SIZE + payloadLength); + const payloadCrc = view.getUint32(16, false); + const computedPayloadCrc = crc32(payload); + if (payloadCrc !== computedPayloadCrc) { + return { ok: false, reason: `payload CRC mismatch (stored=${payloadCrc} computed=${computedPayloadCrc})` }; + } + + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(payload)); + } catch (error) { + return { ok: false, reason: `payload is not valid JSON: ${(error as Error).message}` }; + } + const validated = validateManifestShape(parsed, generation); + if (!validated.ok) return validated; + return { ok: true, manifest: validated.manifest }; + } catch (error) { + return { ok: false, reason: `decode threw: ${(error as Error).message}` }; + } +} + +// --------------------------------------------------------------------------- +// 序列化 / 校验 +// --------------------------------------------------------------------------- + +/** 序列化:只保留有意义的字段,并把 Uint8Array 类字段规整掉(manifest 必须是 JSON 可表示的) */ +function serializeManifest(manifest: AriaManifest): AriaManifest { + const namespaces: Record = {}; + for (const [ns, state] of Object.entries(manifest.namespaces)) { + namespaces[ns] = { + nextSstableId: state.nextSstableId, + sstables: state.sstables.map((m) => ({ + id: m.id, + level: m.level, + minKey: m.minKey, + maxKey: m.maxKey, + blockCount: m.blockCount, + totalSize: m.totalSize, + // bloom 数据当前恒为 null;若被塞入 Uint8Array 则由 LSM 侧保证不为 manifest 内容 + bloomData: null, + ...(m.pageIds && m.pageIds.length > 0 ? { pageIds: [...m.pageIds] } : {}), + })), + }; + } + return { + formatVersion: manifest.formatVersion, + generation: manifest.generation, + pageIdWatermark: manifest.pageIdWatermark, + namespaces, + schemas: manifest.schemas, + wal: { + startSegment: manifest.wal.startSegment, + startLsn: manifest.wal.startLsn, + nextLsn: manifest.wal.nextLsn, + }, + frozen: manifest.frozen.map((f) => ({ ...f })), + owner: { ...manifest.owner }, + committedAt: manifest.committedAt, + }; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function toNonNegativeInt(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || !Number.isInteger(value)) { + throw new Error(`manifest field "${field}" must be a non-negative integer, got ${JSON.stringify(value)}`); + } + return value; +} + +function toNonEmptyString(value: unknown, field: string): string { + if (typeof value !== 'string') { + throw new Error(`manifest field "${field}" must be a string, got ${JSON.stringify(value)}`); + } + return value; +} + +/** 严格形状校验:任何不符合的类型都判该世代无效(而不是"部分采用") */ +function validateManifestShape(value: unknown, generation: number): ManifestDecodeResult { + try { + if (!isPlainObject(value)) return { ok: false, reason: 'payload is not an object' }; + const formatVersion = toNonNegativeInt(value.formatVersion, 'formatVersion'); + if (formatVersion !== MANIFEST_FORMAT_VERSION) { + return { ok: false, reason: `payload formatVersion ${formatVersion}` }; + } + const payloadGeneration = toNonNegativeInt(value.generation, 'generation'); + if (payloadGeneration !== generation) { + return { ok: false, reason: `generation mismatch (header=${generation} payload=${payloadGeneration})` }; + } + + const pageIdWatermark = toNonNegativeInt(value.pageIdWatermark, 'pageIdWatermark'); + + // ---- namespaces ---- + if (!isPlainObject(value.namespaces)) { + return { ok: false, reason: 'namespaces is not an object' }; + } + const namespaces: Record = {}; + for (const [ns, raw] of Object.entries(value.namespaces)) { + if (!isPlainObject(raw)) return { ok: false, reason: `namespace "${ns}" is not an object` }; + const nextSstableId = toNonNegativeInt(raw.nextSstableId, `namespaces.${ns}.nextSstableId`); + if (!Array.isArray(raw.sstables)) { + return { ok: false, reason: `namespaces.${ns}.sstables is not an array` }; + } + const sstables: SSTableMeta[] = []; + for (const item of raw.sstables) { + if (!isPlainObject(item)) return { ok: false, reason: `namespace "${ns}" has a non-object sstable` }; + const id = toNonNegativeInt(item.id, `namespaces.${ns}.sstables[].id`); + const level = toNonNegativeInt(item.level, `namespaces.${ns}.sstables[].level`); + const blockCount = toNonNegativeInt(item.blockCount, `namespaces.${ns}.sstables[].blockCount`); + const totalSize = toNonNegativeInt(item.totalSize, `namespaces.${ns}.sstables[].totalSize`); + const minKey = toNonEmptyString(item.minKey, `namespaces.${ns}.sstables[].minKey`); + const maxKey = toNonEmptyString(item.maxKey, `namespaces.${ns}.sstables[].maxKey`); + let pageIds: number[] | undefined; + if (item.pageIds !== undefined) { + if (!Array.isArray(item.pageIds)) { + return { ok: false, reason: `namespaces.${ns}.sstables[].pageIds is not an array` }; + } + pageIds = item.pageIds.map((pid, i) => + toNonNegativeInt(pid, `namespaces.${ns}.sstables[].pageIds[${i}]`)); + } + sstables.push({ + id, + level, + minKey, + maxKey, + blockCount, + totalSize, + bloomData: null, + ...(pageIds ? { pageIds } : {}), + }); + } + namespaces[ns] = { nextSstableId, sstables }; + } + + // ---- schemas ---- + if (!isPlainObject(value.schemas)) { + return { ok: false, reason: 'schemas is not an object' }; + } + const schemas: Record> = {}; + for (const [table, cols] of Object.entries(value.schemas)) { + if (!isPlainObject(cols)) return { ok: false, reason: `schemas.${table} is not an object` }; + schemas[table] = cols as Record; + } + + // ---- wal ---- + if (!isPlainObject(value.wal)) return { ok: false, reason: 'wal is not an object' }; + const wal: ManifestWalState = { + startSegment: toNonNegativeInt(value.wal.startSegment, 'wal.startSegment'), + startLsn: toNonNegativeInt(value.wal.startLsn, 'wal.startLsn'), + nextLsn: toNonNegativeInt(value.wal.nextLsn, 'wal.nextLsn'), + }; + if (wal.startLsn > wal.nextLsn) { + return { ok: false, reason: `wal.startLsn ${wal.startLsn} > wal.nextLsn ${wal.nextLsn}` }; + } + + // ---- frozen ---- + if (!Array.isArray(value.frozen)) return { ok: false, reason: 'frozen is not an array' }; + const frozen: ManifestFrozenIntent[] = []; + for (const item of value.frozen) { + if (!isPlainObject(item)) return { ok: false, reason: 'frozen has a non-object entry' }; + frozen.push({ + ns: toNonEmptyString(item.ns, 'frozen[].ns'), + id: toNonNegativeInt(item.id, 'frozen[].id'), + entryCount: toNonNegativeInt(item.entryCount, 'frozen[].entryCount'), + minKey: toNonEmptyString(item.minKey, 'frozen[].minKey'), + maxKey: toNonEmptyString(item.maxKey, 'frozen[].maxKey'), + lsnAtFreeze: toNonNegativeInt(item.lsnAtFreeze, 'frozen[].lsnAtFreeze'), + }); + } + + // ---- owner ---- + if (!isPlainObject(value.owner)) return { ok: false, reason: 'owner is not an object' }; + const owner: ManifestOwner = { + instanceId: toNonEmptyString(value.owner.instanceId, 'owner.instanceId'), + epoch: toNonNegativeInt(value.owner.epoch, 'owner.epoch'), + openedAt: toNonNegativeInt(value.owner.openedAt, 'owner.openedAt'), + }; + + const committedAt = toNonNegativeInt(value.committedAt, 'committedAt'); + + return { + ok: true, + manifest: { + formatVersion, + generation, + pageIdWatermark, + namespaces, + schemas, + wal, + frozen, + owner, + committedAt, + }, + }; + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } +} + +// --------------------------------------------------------------------------- +// ManifestStore +// --------------------------------------------------------------------------- + +export interface ManifestStoreOptions { + backend: IStorageBackend; + /** 本实例 id(陈旧实例保护用;默认随机生成) */ + instanceId?: string; + /** 时间源(测试可注入) */ + now?: () => number; +} + +let instanceCounter = 0; + +/** + * manifest 读写器:加载最新有效世代、串行提交、维护回退窗口。 + * + * 注意:本类**不做数据落盘**,它只负责"提交点"。调用方必须先让数据真实落盘 + * (SSTable 页面 flush 完成),再 `commit()`。 + */ +export class ManifestStore { + private backend: IStorageBackend; + private now: () => number; + private state: AriaManifest; + private loaded = false; + private hadInvalidGenerations = false; + private skipped: { generation: number; reason: string }[] = []; + /** 提交串行链:把 manifest 写入序列化成严格顺序 */ + private commitChain: Promise = Promise.resolve(); + /** 已提交的世代号(= 最新有效世代) */ + private generation = 0; + + constructor(opts: ManifestStoreOptions) { + this.backend = opts.backend; + this.now = opts.now ?? (() => Date.now()); + this.state = createEmptyManifest({ + instanceId: opts.instanceId ?? `aria-${++instanceCounter}-${Math.random().toString(36).slice(2, 10)}`, + now: this.now(), + }); + } + + /** 本实例 id */ + get instanceId(): string { + return this.state.owner.instanceId; + } + + /** 当前内存态(调用方可直接修改;提交通过 `commit()` 串行化) */ + get current(): AriaManifest { + return this.state; + } + + /** 最近一次加载/提交后的世代号 */ + get currentGeneration(): number { + return this.generation; + } + + /** + * 加载最新有效世代。 + * + * - 没有任何 manifest 文件 → `{ manifest: null }`(全新库); + * - 有文件且最新世代有效 → 采用它(更早的损坏世代只记录、不删除); + * - 有文件但**全部世代无效** → 抛 `ARIA_MANIFEST_CORRUPT` + * (绝不返回空状态:那会让上层把"元数据全坏"当成"空库", + * 随后 repair 还会把没人引用的活页删干净 —— 不可逆)。 + */ + async load(): Promise { + const keys = await this.backend.listKeys(); + const generations = keys + .map((k) => generationFromKey(k)) + .filter((g): g is number => g !== null) + .sort((a, b) => b - a); + + const skipped: { generation: number; reason: string }[] = []; + for (const gen of generations) { + const raw = await this.backend.read(manifestKey(gen)); + if (!raw) { + skipped.push({ generation: gen, reason: 'manifest file disappeared during load' }); + continue; + } + const decoded = decodeManifest(new Uint8Array(raw)); + if (!decoded.ok) { + skipped.push({ generation: gen, reason: decoded.reason }); + continue; + } + if (decoded.manifest.generation !== gen) { + skipped.push({ + generation: gen, + reason: `file name generation ${gen} != payload generation ${decoded.manifest.generation}`, + }); + continue; + } + this.state = decoded.manifest; + this.generation = gen; + this.skipped = skipped; + this.hadInvalidGenerations = skipped.length > 0; + this.loaded = true; + return { manifest: decoded.manifest, generation: gen, hadInvalidGenerations: this.hadInvalidGenerations, skipped }; + } + + if (generations.length > 0) { + // 文件存在但一代都读不出来:这是元数据损坏,不是空库 + throw new DatabaseError( + `AriaEngine manifest is corrupt: ${generations.length} generation(s) present, none passed validation ` + + `(${skipped.map((s) => `gen ${s.generation}: ${s.reason}`).join('; ')})`, + 'ARIA_MANIFEST_CORRUPT', + ); + } + + this.loaded = true; + return { manifest: null, generation: 0, hadInvalidGenerations: false, skipped: [] }; + } + + /** 用一份外部状态替换内存态(旧格式迁移 / 测试用),不落盘 */ + adopt(manifest: AriaManifest): void { + this.state = manifest; + this.generation = manifest.generation; + } + + /** + * 认领所有权:把当前内存态提交为新世代(owner 换成本实例)。 + * + * 与普通 `commit()` 的区别:这里**允许**接手别人提交的世代 + * (多标签页的互斥由 Web Locks 负责;即便没有 Web Locks,接手者也只会让 + * 旧实例后续提交被拒绝,而不是让旧实例静默覆盖新数据)。 + */ + async claimOwnership(): Promise { + return this.commitInternal({ allowTakeover: true }); + } + + /** + * 提交当前内存态为新世代。 + * + * 提交前会**重新读取磁盘上的最新世代号**:若它已经超过本实例上次提交的世代, + * 说明另一个实例在我们不知情的情况下提交过(陈旧实例)—— 此时抛 + * `STALE_INSTANCE`,而不是用陈旧的内存态覆盖新一代(修复前 KVStore 快照被 + * 静默覆盖、Aria 侧无任何保护)。 + */ + async commit(): Promise { + return this.commitInternal({ allowTakeover: false }); + } + + private async commitInternal(opts: { allowTakeover: boolean }): Promise { + let result!: AriaManifest; + let failure: unknown = null; + const run = this.commitChain.then(async () => { + try { + result = await this.doCommit(opts); + } catch (error) { + failure = error; + } + }); + this.commitChain = run; + await run; + if (failure) throw failure; + return result; + } + + private async doCommit(opts: { allowTakeover: boolean }): Promise { + if (!this.loaded) { + throw new DatabaseError('ManifestStore.commit() before load()', 'ARIA_MANIFEST_NOT_LOADED'); + } + const onDisk = await this.newestGenerationOnDisk(); + // 陈旧实例判定要**跳过已知损坏的世代**:损坏世代(撕裂写/介质坏块)不是 + // "另一个实例提交的状态",把它算进来会让库永远无法再提交(实测: + // 一个损坏的更高世代把后续所有提交都拦成 STALE_INSTANCE)。 + const knownInvalid = new Set(this.skipped.map((s) => s.generation)); + let newestForeign = 0; + for (const gen of await this.listGenerationNumbers()) { + if (gen > this.generation && !knownInvalid.has(gen)) newestForeign = Math.max(newestForeign, gen); + } + if (!opts.allowTakeover && newestForeign > this.generation) { + throw new DatabaseError( + `Refusing to commit: another instance committed generation ${newestForeign} ` + + `(this instance last committed ${this.generation}) — stale instance`, + 'STALE_INSTANCE', + ); + } + + // 但世代号必须大于**任何**已存在的文件(含损坏世代),否则会覆盖它。 + // 认领(takeover)时一次跨过一段:见 MANIFEST_TAKEOVER_STRIDE 的说明。 + const nextGeneration = opts.allowTakeover + ? onDisk + MANIFEST_TAKEOVER_STRIDE + : onDisk + 1; + const next: AriaManifest = { + ...this.state, + formatVersion: MANIFEST_FORMAT_VERSION, + generation: nextGeneration, + namespaces: this.state.namespaces, + schemas: this.state.schemas, + wal: { ...this.state.wal }, + frozen: this.state.frozen.map((f) => ({ ...f })), + owner: { + instanceId: this.state.owner.instanceId, + epoch: this.state.owner.epoch + 1, + openedAt: this.state.owner.openedAt, + }, + committedAt: this.now(), + }; + + const encoded = encodeManifest(next); + await this.backend.write(manifestKey(nextGeneration), encoded.buffer.slice( + encoded.byteOffset, + encoded.byteOffset + encoded.byteLength, + ) as ArrayBuffer); + + // 先写后验:回读并校验,只有真正可读回的新世代才算提交成功 + const readBack = await this.backend.read(manifestKey(nextGeneration)); + if (!readBack) { + throw new DatabaseError( + `Manifest commit verification failed: generation ${nextGeneration} not readable after write`, + 'ARIA_MANIFEST_WRITE_FAILED', + ); + } + const verified = decodeManifest(new Uint8Array(readBack)); + if (!verified.ok || verified.manifest.generation !== nextGeneration) { + throw new DatabaseError( + `Manifest commit verification failed: generation ${nextGeneration} ` + + `(${verified.ok ? 'generation mismatch' : verified.reason})`, + 'ARIA_MANIFEST_WRITE_FAILED', + ); + } + + this.state = next; + this.generation = nextGeneration; + + // 回退窗口之外的历史世代:只有在**没有损坏世代**时才清理 + //(存在损坏世代时保留全部,绝不因为"读不出来"就删掉可能是唯一副本的东西) + if (!this.hadInvalidGenerations) { + await this.pruneOldGenerations(nextGeneration); + } + return next; + } + + /** 磁盘上最大的 manifest 世代号(乐观并发检查;只读 key 名单) */ + async newestGenerationOnDisk(): Promise { + return (await this.listGenerationNumbers()).reduce((max, g) => Math.max(max, g), 0); + } + + /** 磁盘上全部 manifest 世代号(升序无关,仅用于判定) */ + private async listGenerationNumbers(): Promise { + const keys = await this.backend.listKeys(); + const out: number[] = []; + for (const k of keys) { + const gen = generationFromKey(k); + if (gen !== null) out.push(gen); + } + return out; + } + + /** 删除回退窗口之外的世代(当前 + 上一代保留) */ + private async pruneOldGenerations(currentGeneration: number): Promise { + const keys = await this.backend.listKeys(); + const stale = keys + .map((k) => generationFromKey(k)) + .filter((g): g is number => g !== null && g < currentGeneration - (MANIFEST_RETAIN_GENERATIONS - 1)); + if (stale.length === 0) return; + try { + await this.backend.deleteMany(stale.map((g) => manifestKey(g))); + } catch { + // 删除失败只是残留文件(下次提交再清),不影响正确性 + } + } +} diff --git a/src/engine/aria/store/page_sstable_store.ts b/src/engine/aria/store/page_sstable_store.ts index 70a879e..10fcf3d 100644 --- a/src/engine/aria/store/page_sstable_store.ts +++ b/src/engine/aria/store/page_sstable_store.ts @@ -23,6 +23,18 @@ import { compressLZ4, decompressLZ4 } from '../compression/lz4'; export class PageSSTableStore { /** SSTable id → 页面 ID 列表(save 时记录,saveMeta 时注入 meta) */ private pageIds = new Map(); + /** + * v0.8.0(B-6):已退休(被 compaction 取代 / 已被 manifest 摘除)但仍可能有 + * 在途读者持有引用的 SSTable → 页面 ID 列表。 + * + * 为什么必须保留:读路径是"先取 meta 快照、再按 id 加载数据",快照与加载之间 + * 可以插入一次 compaction。若退休时立刻忘掉 pageIds,在途读者的 `load()` 就 + * 找不到页面,只能把"文件已退休"误判为"数据缺失"(旧代码会顺手删掉 meta 并 + * 打一条损坏告警)。保留到物理删除为止,语义才是自洽的。 + */ + private retiredPageIds = new Map(); + /** SSTable id → 落盘字节数(load 时截断最后一页 0 填充) */ + private storedSizes = new Map(); constructor( private fileManager: FileManager, @@ -72,24 +84,47 @@ export class PageSSTableStore { this.bufferPool.unpin(page); } this.pageIds.set(id, ids); + this.retiredPageIds.delete(id); + this.storedSizes.set(id, payload.byteLength); return { storedSize: payload.byteLength }; } - /** 获取指定 SSTable 的页面 ID 列表(saveMeta 注入用) */ + /** 获取指定 SSTable 的页面 ID 列表(saveMeta 注入用;未注册返回 undefined) */ getPageIds(id: number): number[] | undefined { - return this.pageIds.get(id); + return this.pageIds.get(id) ?? this.retiredPageIds.get(id); + } + + /** v0.8.0:打开时从 manifest 把已有 SSTable 的页面映射注册进来(load 不再依赖调用方传参) */ + registerPageIds(id: number, ids: number[], storedSize?: number): void { + if (this.pageIds.has(id)) return; + this.pageIds.set(id, [...ids]); + if (typeof storedSize === 'number') this.storedSizes.set(id, storedSize); + } + + /** + * v0.8.0:标记某 SSTable 已退休(被合并产物取代)。 + * 页面映射保留,在途读者仍能读到旧数据;物理删除由 `delete()` 完成。 + */ + retirePageIds(id: number): void { + const ids = this.pageIds.get(id); + if (!ids) return; + this.pageIds.delete(id); + this.retiredPageIds.set(id, ids); } /** * 按页面 ID 列表读取并拼接为完整字节流。 - * @param totalSize 页面中**实际存储**的字节数(`save()` 返回的 storedSize, - * 即压缩后长度)——最后一页可能有 0 填充,按它截断。 + * @param pageIds 页面 ID 列表(缺省时用内部注册的映射) + * @param totalSize 页面中**实际存储**的字节数(缺省时用内部记录) * @returns 缺失页面/读取失败返回 null(调用方视为损坏并清理) */ - async load(id: number, pageIds: number[], totalSize: number): Promise { - if (pageIds.length === 0) return null; + async load(id: number, pageIds?: number[], totalSize?: number): Promise { + const ids = pageIds ?? this.pageIds.get(id) ?? this.retiredPageIds.get(id); + if (!ids || ids.length === 0) return null; + const size = totalSize ?? this.storedSizes.get(id); + if (typeof size !== 'number') return null; const chunks: Uint8Array[] = []; - for (const pageId of pageIds) { + for (const pageId of ids) { const page = await this.bufferPool.getPage(pageId); if (!page) return null; // 立即复制(后续驱逐安全) @@ -97,7 +132,7 @@ export class PageSSTableStore { this.bufferPool.unpin(page); } const total = chunks.reduce((s, c) => s + c.byteLength, 0); - const out = new Uint8Array(Math.min(total, totalSize)); + const out = new Uint8Array(Math.min(total, size)); let off = 0; for (const c of chunks) { const take = Math.min(c.byteLength, out.byteLength - off); @@ -105,19 +140,23 @@ export class PageSSTableStore { out.set(c.subarray(0, take), off); off += take; } - this.pageIds.delete(id); + // v0.8.0:**不再**在这里丢掉 pageIds —— 退休 SSTable 的在途读者仍会调用 load, + // 丢掉映射会让它们把"已退休"误判成"数据缺失"。物理删除由 delete() 负责。 // v0.8.0(A38):解压(与 save 的加密/压缩顺序对称) return this.compression ? decompressLZ4(out) : out; } - /** 释放页面(删除物理页面文件 + 移出 BufferPool) */ - async delete(id: number, pageIds: number[]): Promise { - for (const pageId of pageIds) { + /** 释放页面(删除物理页面文件 + 移出 BufferPool;同时清掉活跃与退休映射) */ + async delete(id: number, pageIds?: number[]): Promise { + const ids = pageIds ?? this.pageIds.get(id) ?? this.retiredPageIds.get(id) ?? []; + for (const pageId of ids) { this.bufferPool.removePage(pageId); try { await this.fileManager.freePageId(pageId); } catch { /* 清理失败不阻塞 */ } } this.pageIds.delete(id); + this.retiredPageIds.delete(id); + this.storedSizes.delete(id); } } diff --git a/src/engine/aria/wal/checkpoint.ts b/src/engine/aria/wal/checkpoint.ts index 257c899..b3e8563 100644 --- a/src/engine/aria/wal/checkpoint.ts +++ b/src/engine/aria/wal/checkpoint.ts @@ -12,6 +12,18 @@ import type { WAL } from './log'; export interface Flushable { flushAll(): Promise; + /** + * v0.8.0(B-6/55):**只落 memtable**(不等待 compaction)。 + * + * 为什么必须分开:checkpoint 要保证的是"WAL 覆盖的数据已落盘",而 compaction + * 只是重排**已经落盘**的 SSTable。修复前 checkpoint 会先 `lsm.flush()` 再 + * `flushable.flushAll()`,两者都会排空后台维护链 —— 后台 compaction 跑几秒, + * 写路径每 1000 次操作就要等它一次(v0.6.1 记录的 "8~11s 悬崖"的另一半)。 + * + * 未实现该方法的调用方退回旧语义(先 flush LSM 再 flushAll), + * 因此这个接口对既有测试替身保持兼容。 + */ + flushMemtables?(): Promise; } // --------------------------------------------------------------------------- @@ -60,9 +72,15 @@ export class CheckpointManager { } async checkpoint(): Promise { - await this.lsm.flush(); - if (this.flushable) { - await this.flushable.flushAll(); + if (this.flushable && typeof this.flushable.flushMemtables === 'function') { + // v0.8.0(B-6/55):只等 memtable 落盘;compaction 继续在后台跑 + await this.flushable.flushMemtables(); + } else { + // 兼容路径(测试替身 / 未实现新接口的调用方):旧语义 + await this.lsm.flush(); + if (this.flushable) { + await this.flushable.flushAll(); + } } await this.wal.checkpoint(); this.opCount = 0; diff --git a/src/engine/aria/wal/log.ts b/src/engine/aria/wal/log.ts index 42f7cbb..96e7f1e 100644 --- a/src/engine/aria/wal/log.ts +++ b/src/engine/aria/wal/log.ts @@ -20,8 +20,10 @@ * └──────────┴──────────────┴──────────┘ */ +import { DatabaseError } from '../../../constants'; import { WALRecordType, type WALRecord } from '../types'; import { crc32 } from '../crc32'; +import type { WALReadResult } from './segmented_store'; // --------------------------------------------------------------------------- // WAL 存储接口 @@ -36,6 +38,61 @@ export interface WALStore { truncate(): Promise; /** 检查 WAL 是否存在 */ exists(): Promise; + /** + * v0.8.0(可选,分片存储实现):从指定分片起读取,并把空洞如实返回。 + * 未实现的 store 由 WAL 回退为 `readAll()`(无空洞信息)。 + */ + readAllFrom?(fromSegment: number): Promise; + /** + * v0.8.0(可选,分片存储实现):删除整体已落盘的前缀分片, + * 返回仍需保留的最小分片号。未实现时 WAL 只在"全部已落盘"时整体截断。 + * + * @param latestLsn 当前 LSN 高水位(判断"最后一个分片"是否也被完整覆盖) + */ + truncateBefore?(durableLsn: number, latestLsn?: number): Promise; + /** + * v0.8.0(可选):整库清空后重置分片编号(仅 `clearAll` 这类"介质整体抹掉、 + * manifest 也从 0 开始"的场景)。 + */ + reset?(): void; + /** + * v0.8.0(可选,分片存储实现):**只计算**仍需保留的最小分片号(无副作用)。 + * 调用方必须先把它提交进 manifest,再调用 `truncateBefore` 删除 —— 顺序反了 + * 会让恢复无法区分"正常清理过的前缀"与"介质丢了一段记录"。 + */ + planKeepFrom?(durableLsn: number, latestLsn?: number): Promise; +} + +/** WAL 恢复的边界选项(v0.8.0) */ +export interface WALRecoverOptions { + /** + * 跳过 `lsn <= fromLsn` 的记录。 + * + * 这些记录已由 manifest 证明存在于已提交的 SSTable 中(`wal.startLsn`), + * 重放它们不仅浪费启动时间,还会把**已经删除的数据复活** + *(旧记录晚于 tombstone 重放时)。 + */ + fromLsn?: number; + /** 从该分片号开始读取(manifest 的 `wal.startSegment`) */ + fromSegment?: number; + /** + * 允许活跃区间内的分片空洞。默认 false → 抛 `ARIA_WAL_GAP`。 + * + * 为什么默认抛错:空洞意味着"某个已提交事务的记录缺失",静默继续会把 + * 不完整的状态当成完整状态。引擎显式传 `allowGaps: true` 并把它写进 + * 恢复报告(用户可见),而不是让调用方悄悄少一批数据。 + */ + allowGaps?: boolean; +} + +/** 最近一次恢复的诊断信息 */ +export interface WALRecoveryInfo { + applied: number; + skipped: number; + maxLsn: number; + gaps: number[]; + fromSegment: number; + fromLsn: number; } // --------------------------------------------------------------------------- @@ -50,6 +107,8 @@ export class WAL { private syncMode: 'full' | 'batch' | 'none'; /** v0.3.3: 未 checkpoint 的 WAL 累计字节数(full/batch/none 通用) */ private bufferedBytes = 0; + /** v0.8.0: 最近一次恢复诊断 */ + private lastRecoveryInfo: WALRecoveryInfo | null = null; constructor(store: WALStore, enabled: boolean = true, syncMode: 'full' | 'batch' | 'none' = 'batch') { this.store = store; @@ -137,22 +196,72 @@ export class WAL { /** 从 WAL 恢复未提交的事务数据 */ async recover( applyRecord: (record: WALRecord) => void, + opts: WALRecoverOptions = {}, ): Promise { if (!this.enabled) return 0; - const exists = await this.store.exists(); - if (!exists) return 0; + const fromSegment = opts.fromSegment ?? 0; + const fromLsn = opts.fromLsn ?? 0; + let data: Uint8Array; + let gaps: number[] = []; - const data = await this.store.readAll(); - if (data.byteLength === 0) return 0; - - const records = this.decodeAllRecords(data); - for (const record of records) { - applyRecord(record); + if (typeof this.store.readAllFrom === 'function') { + const result = await this.store.readAllFrom(fromSegment); + data = result.data; + gaps = result.gaps; + } else { + const exists = await this.store.exists(); + if (!exists) return 0; + data = await this.store.readAll(); } - this.lsn = records.length > 0 ? records[records.length - 1].lsn : 0; - return records.length; + if (gaps.length > 0 && !opts.allowGaps) { + throw new DatabaseError( + `WAL segment gap detected in live range (missing segment(s): ${gaps.join(', ')}) — ` + + 'records after the gap cannot be verified; refusing to continue silently', + 'ARIA_WAL_GAP', + ); + } + + const records = data.byteLength === 0 ? [] : this.decodeAllRecords(data); + let applied = 0; + let skipped = 0; + let maxLsn = 0; + for (const record of records) { + if (record.lsn > maxLsn) maxLsn = record.lsn; + // v0.8.0: lsn <= startLsn 的记录已由 manifest 证明落盘,跳过(不再重复重放) + if (record.lsn <= fromLsn) { + skipped++; + continue; + } + applyRecord(record); + applied++; + } + + this.lsn = Math.max(this.lsn, maxLsn); + // 磁盘上的全部记录已经读出来了,缓冲区里不再有待落盘记录 + this.buffer = []; + this.bufferedBytes = 0; + this.lastRecoveryInfo = { applied, skipped, maxLsn, gaps, fromSegment, fromLsn }; + return applied; + } + + /** v0.8.0: 最近一次恢复诊断(applied/skipped/gaps) */ + getLastRecoveryInfo(): WALRecoveryInfo | null { + return this.lastRecoveryInfo ? { ...this.lastRecoveryInfo, gaps: [...this.lastRecoveryInfo.gaps] } : null; + } + + /** v0.8.0: 当前 LSN 高水位(manifest 记录它以保证重启后 LSN 继续单调) */ + getLsn(): number { + return this.lsn; + } + + /** + * v0.8.0: 从 manifest 的高水位继续 LSN(重启后 LSN 全库单调,永不复用)。 + * 只允许上调,不允许回退。 + */ + setLsn(lsn: number): void { + if (Number.isFinite(lsn) && lsn > this.lsn) this.lsn = lsn; } // ======================================================================= @@ -164,8 +273,50 @@ export class WAL { if (!this.enabled) return; await this.flush(); await this.store.truncate(); - this.lsn = 0; this.bufferedBytes = 0; + // v0.8.0: **不再把 lsn 归零**。LSN 是 manifest 记录的全库单调水位 + // (`wal.nextLsn`),归零会让"跳过 lsn <= startLsn"的判定与历史分片冲突: + // 同一段 LSN 区间会对应两批完全不同的记录。 + } + + /** v0.8.0:整库清空后重置分片编号(供 `clearAll` 使用) */ + reset(): void { + if (typeof this.store.reset === 'function') this.store.reset(); + } + + /** + * v0.8.0(B-6):查询"提交之后可以删到哪个分片"(无副作用)。 + * @returns 仍需保留的最小分片号(未实现分片能力的 store 返回 0 = 全部保留) + */ + async planKeepFromSegment(durableLsn: number): Promise { + if (!this.enabled) return 0; + if (typeof this.store.planKeepFrom === 'function') { + return this.store.planKeepFrom(durableLsn, this.lsn); + } + return 0; + } + + /** + * v0.8.0(B-6):**按落盘水位**截断 WAL —— manifest 提交之后的清理动作。 + * + * 与 `checkpoint()` 的区别:这里只删除"整段记录都 <= durableLsn"的前缀分片, + * 因此可以安全地在**后台 compaction 仍在进行**时调用。 + * + * @returns 仍需保留的最小分片号(调用方应写入 manifest 的 `wal.startSegment`) + */ + async checkpointBefore(durableLsn: number): Promise { + if (!this.enabled) return 0; + // 先把缓冲记录落盘:否则"边界分片"的判定会漏掉刚写进缓冲的记录 + await this.flush(); + if (typeof this.store.truncateBefore === 'function') { + return this.store.truncateBefore(durableLsn, this.lsn); + } + // 回退(无分片能力的 store):只有确认"全部记录都已落盘"才整体截断 + if (durableLsn >= this.lsn) { + await this.store.truncate(); + this.bufferedBytes = 0; + } + return 0; } // ======================================================================= diff --git a/src/engine/aria/wal/segmented_store.ts b/src/engine/aria/wal/segmented_store.ts index 249c17c..3f25c52 100644 --- a/src/engine/aria/wal/segmented_store.ts +++ b/src/engine/aria/wal/segmented_store.ts @@ -16,7 +16,9 @@ import type { IStorageBackend } from '../store/backend'; /** 分片文件名:__wal_%06d.bin */ export const WAL_SEGMENT_PREFIX = '__wal_'; -const SEGMENT_REGEX = /^__wal_(\d{6})\.bin$/; +// v0.8.0:分片号只增不减(见 truncateBefore),因此不能假设一定是 6 位 —— +// 超过 999999 之后若仍按 {6} 匹配,分片会突然"消失"(静默丢日志)。 +const SEGMENT_REGEX = /^__wal_(\d{6,})\.bin$/; const LEGACY_RECORD_REGEX = /^__wal_(\d+)$/; const LEGACY_COUNT_KEY = '__wal_count'; @@ -34,6 +36,24 @@ export interface WALStore { exists(): Promise; } +/** 读取结果:数据 + 空洞诊断(v0.8.0) */ +export interface WALReadResult { + /** 有效前缀数据(自 fromSegment 起、连续分片拼接) */ + data: Uint8Array; + /** 起始分片号 */ + fromSegment: number; + /** 实际读到的分片号(升序) */ + segments: number[]; + /** + * 活跃区间内的空洞(缺失的分片号)。非空意味着"这之后的记录不可信" —— + * v0.8.0 起由调用方(WAL/引擎)**显式上报**,不再静默丢弃尾部 + * (修复前:序号不连续 → 直接把空洞之后的分片全部丢掉且没有任何提示)。 + */ + gaps: number[]; + /** 最后一条记录之后仍缺失的分片数(尾部截断诊断) */ + missingTail: number; +} + // --------------------------------------------------------------------------- // SegmentedWALStore // --------------------------------------------------------------------------- @@ -44,6 +64,14 @@ export class SegmentedWALStore implements WALStore { private currentSegment = 0; /** 当前分片字节数(内存跟踪,append 切分片判断) */ private currentSize = 0; + /** + * v0.8.0:分片 → 该分片**首条记录**的 LSN。 + * + * 用途:manifest 提交后需要知道"哪些分片整体已落盘可以删除"。 + * 记录格式里 LSN 是每条记录的第 1 个字段,因此取分片前 4 字节即可定位; + * 无需改变 WAL 二进制格式(旧库分片同样适用)。 + */ + private segmentFirstLsn = new Map(); constructor( private backend: IStorageBackend, @@ -64,9 +92,18 @@ export class SegmentedWALStore implements WALStore { this.currentSegment++; this.currentSize = 0; } - const key = this.segmentKey(this.currentSegment); + const seq = this.currentSegment; + const key = this.segmentKey(seq); const copy = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer; + // 记录该分片首条记录的 LSN(分片内记录按写入顺序追加,首条即最小 LSN) + if (!this.segmentFirstLsn.has(seq) && data.byteLength >= 4) { + this.segmentFirstLsn.set( + seq, + new DataView(data.buffer, data.byteOffset, 4).getUint32(0, false), + ); + } + if (typeof this.backend.append === 'function') { await this.backend.append!(key, copy); } else { @@ -84,22 +121,48 @@ export class SegmentedWALStore implements WALStore { this.currentSize += data.byteLength; } - async readAll(): Promise { + /** 列出存储上的分片(升序) */ + private async listSegments(): Promise<{ seq: number; key: string }[]> { const keys = await this.backend.listKeys(); - - // ---- 新格式分片 ---- - const segments = keys + return keys .filter((k) => SEGMENT_REGEX.test(k)) .map((k) => ({ seq: Number(k.match(SEGMENT_REGEX)![1]), key: k })) .sort((a, b) => a.seq - b.seq); + } - // 空洞检测:分片序号必须从 0 严格连续,空洞后的分片整体丢弃 - let keepCount = 0; - for (let i = 0; i < segments.length; i++) { - if (segments[i].seq !== i) break; - keepCount = i + 1; + /** + * v0.8.0:从 `fromSegment` 开始读取,并把空洞**如实返回**(不再静默丢弃尾部)。 + * + * 为什么必须报告空洞:`truncateBefore` 只会删除**前缀**分片,因此活跃区间内 + * 出现空洞只可能来自介质损坏/外部删除 —— 此时空洞之后的记录无法确认是否属于 + * 同一个连续历史。修复前 `readAll` 遇到空洞直接丢弃空洞之后的全部记录, + * 调用方(引擎恢复)完全无法感知"少了一批已提交事务"。 + */ + async readAllFrom(fromSegment: number = 0): Promise { + const keys = await this.backend.listKeys(); + const allSegments = await this.listSegments(); + const segments = allSegments.filter((s) => s.seq >= fromSegment); + + // 空洞检测:活跃区间内分片序号必须连续。 + // v0.8.0:**前缀缺失**同样算空洞 —— 调用方(manifest 的 `wal.startSegment`) + // 保证"小于 startSegment 的分片已整体落盘、可以不存在",因此活跃区间内 + // 第一片就缺席(segments[0].seq > fromSegment)意味着那段记录不知所踪。 + const gaps: number[] = []; + if (segments.length > 0) { + for (let missing = fromSegment; missing < segments[0].seq; missing++) gaps.push(missing); + } + for (let i = 1; i < segments.length; i++) { + for (let missing = segments[i - 1].seq + 1; missing < segments[i].seq; missing++) { + gaps.push(missing); + } + } + + // 空洞之后的分片整体丢弃(长度字段链无法跨空洞验证),并计入 diagnostics + let kept = segments; + if (gaps.length > 0) { + const firstGap = gaps[0]; + kept = segments.filter((s) => s.seq < firstGap); } - const validSegments = segments.slice(0, keepCount); // ---- 旧格式兼容:__wal_N 单记录键(迁移前数据) ---- const legacyKeys = keys @@ -114,15 +177,23 @@ export class SegmentedWALStore implements WALStore { if (raw) parts.push(new Uint8Array(raw)); } // 新格式分片在后 - for (const { key } of validSegments) { + const readSegments: number[] = []; + for (const { seq, key } of kept) { const raw = await this.backend.read(key); - if (raw) parts.push(new Uint8Array(raw)); + if (!raw) continue; + const bytes = new Uint8Array(raw); + if (!this.segmentFirstLsn.has(seq) && bytes.byteLength >= 4) { + this.segmentFirstLsn.set(seq, new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0, false)); + } + readSegments.push(seq); + parts.push(bytes); } // 同步当前分片状态(追加定位) - if (validSegments.length > 0) { - this.currentSegment = validSegments[validSegments.length - 1].seq; - const lastRaw = await this.backend.read(validSegments[validSegments.length - 1].key); + if (segments.length > 0) { + const lastSeq = segments[segments.length - 1].seq; + this.currentSegment = lastSeq; + const lastRaw = await this.backend.read(this.segmentKey(lastSeq)); this.currentSize = lastRaw ? lastRaw.byteLength : 0; // 旧格式键存在时(迁移中),下一条记录另起分片,避免与旧键序号冲突 if (legacyKeys.length > 0) { @@ -133,13 +204,28 @@ export class SegmentedWALStore implements WALStore { // 仅有旧格式:迁移中,新写入从分片 0 开始(checkpoint 会清空旧键) this.currentSegment = 0; this.currentSize = 0; + } else if (fromSegment > 0) { + // 活跃区间的分片已被清理(例如上一次 truncateBefore 的删除生效): + // 新写入从当前分片号继续,避免复用已删除的历史分片号 + this.currentSegment = fromSegment; + this.currentSize = 0; } const total = parts.reduce((s, c) => s + c.byteLength, 0); const combined = new Uint8Array(total); let off = 0; for (const c of parts) { combined.set(c, off); off += c.byteLength; } - return combined; + return { + data: combined, + fromSegment, + segments: readSegments, + gaps, + missingTail: 0, + }; + } + + async readAll(): Promise { + return (await this.readAllFrom(0)).data; } async truncate(): Promise { @@ -149,8 +235,142 @@ export class SegmentedWALStore implements WALStore { if (walKeys.length > 0) { await this.backend.deleteMany(walKeys); } + // v0.8.0:分片号只增不减(同 truncateBefore 的说明)—— 复用序号会让 + // "同一序号对应两代记录",恢复按序号排序时旧记录可能排在新记录之后被重放。 + const maxSeq = (await this.listSegments()).reduce((max, s) => Math.max(max, s.seq), -1); + this.currentSegment = Math.max(this.currentSegment, maxSeq + 1); + this.currentSize = 0; + this.segmentFirstLsn.clear(); + } + + /** + * v0.8.0:**整库清空**后重置分片编号(只有 `clearAll` 这种"介质被整体抹掉、 + * manifest 也重新从 0 开始"的场景才允许调用)。 + */ + reset(): void { this.currentSegment = 0; this.currentSize = 0; + this.segmentFirstLsn.clear(); + } + + /** + * v0.8.0:删除**整体 LSN 都 <= durableLsn** 的分片(前缀删除)。 + * + * 语义约束(调用方必须保证):`durableLsn` 之前的记录已存在于已提交的 + * manifest/SSTable 中。返回仍需保留的最小分片号 —— 调用方应把它写进 + * manifest 的 `wal.startSegment`,这样即便删除只完成了一半(崩溃/介质错误), + * 恢复也会忽略那些残留的旧世代分片。 + * + * @param durableLsn 落盘水位(lsn <= 它的记录已确认存在于 SSTable 中) + * @param latestLsn 当前 LSN 高水位(调用方 flush 之后的 `wal.getLsn()`): + * 用于判断**最后一个分片**是否也已被水位完整覆盖。 + * WAL 记录里只有"每条记录的 LSN",分片的**末条** LSN 无法 + * 从分片首字节推出,而"最后一个分片之后没有分片"这一点只有 + * 调用方知道 —— 缺了它就会出现"分片永远删不掉"(实测: + * close 之后 `__wal_000000.bin` 仍在,每次打开都要重读一遍)。 + */ + /** + * v0.8.0:**只计算**"还需要保留的最小分片号"(无副作用)。 + * + * 为什么需要"先算后删":调用方必须**先把这个值提交进 manifest**,再删除分片。 + * 顺序反了(先删后记录)会留下一个窗口:崩溃后 manifest 里的 `startSegment` + * 比介质上实际存在的分片更小,恢复时看到"前缀缺失"就无法区分 + * "正常清理过的前缀"与"介质丢了一段记录"—— 前者无害,后者是数据丢失。 + */ + async planKeepFrom(durableLsn: number, latestLsn?: number): Promise { + const segments = await this.listSegments(); + if (segments.length === 0) return 0; + + // 判定每个分片是否"整段已被水位覆盖": + // - 非末分片:下一分片的首条记录 LSN <= durableLsn ⇒ 本分片全部记录都 <= 水位 + // - 末分片:已知当前 LSN 高水位 <= durableLsn ⇒ 本分片全部记录都 <= 水位 + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + const isLast = i === segments.length - 1; + const firstLsn = this.segmentFirstLsn.get(seg.seq); + // 未知边界:保守地从这里开始保留 + if (firstLsn === undefined) return seg.seq; + const fullyCovered = isLast + ? (typeof latestLsn === 'number' && latestLsn > 0 && latestLsn <= durableLsn) + : (() => { + const nextFirst = this.segmentFirstLsn.get(segments[i + 1].seq); + return nextFirst !== undefined && nextFirst <= durableLsn; + })(); + if (!fullyCovered) return seg.seq; + } + // 全部分片都被水位覆盖 + return segments[segments.length - 1].seq + 1; + } + + async truncateBefore(durableLsn: number, latestLsn?: number): Promise { + // v0.8.0:旧格式(__wal_N 每条一个 key)同样按水位清理 —— + // 它们是"迁移前"的记录,与分片一样只在 lsn <= durableLsn 时才可删。 + // 修复前这些键只在整个 WAL 被 truncate() 时才清,导致已落盘的旧记录 + // 永远留在介质上(每次打开都会被 readAll 读出来再按 lsn 跳过)。 + // + // 注意:必须在"没有新格式分片"的早退**之前**处理 —— 只含旧格式的库 + //(v0.4.4 升级现场)恰恰是这条路径最常见的输入。 + const legacyKeys = (await this.backend.listKeys()).filter((k) => LEGACY_RECORD_REGEX.test(k)); + if (legacyKeys.length > 0) { + const obsoleteLegacy: string[] = []; + for (const key of legacyKeys) { + const raw = await this.backend.read(key); + if (!raw || raw.byteLength < 4) { + obsoleteLegacy.push(key); // 空/残缺:无记录可保留 + continue; + } + const firstLsn = new DataView(raw, 0, 4).getUint32(0, false); + if (firstLsn <= durableLsn) obsoleteLegacy.push(key); + } + if (obsoleteLegacy.length > 0) { + try { + await this.backend.deleteMany([...obsoleteLegacy, LEGACY_COUNT_KEY]); + } catch { /* 同上:残留无害(恢复按 lsn 跳过) */ } + } + } + + const segments = await this.listSegments(); + if (segments.length === 0) { + this.currentSegment = 0; + this.currentSize = 0; + return 0; + } + + const keepFrom = await this.planKeepFrom(durableLsn, latestLsn); + const obsolete = segments.filter((s) => s.seq < keepFrom); + if (obsolete.length > 0) { + try { + await this.backend.deleteMany(obsolete.map((s) => s.key)); + } catch { /* 删除失败:残留分片由 manifest.startSegment 在恢复时忽略 */ } + for (const s of obsolete) this.segmentFirstLsn.delete(s.seq); + } + + // 全部删除(含当前分片)→ **分片号继续往后走,绝不重置回 0**。 + // + // 这是 v0.8.0 的一处关键修正:修复前这里是 `currentSegment = 0`。而 manifest + // 里的 `startSegment` 是**删除前**就算好并提交的(= 最大分片号 + 1),于是 + // 重置之后新记录写进分片 0,恢复时从 startSegment 开始读 → 整批新记录被跳过 + // (实测随机压力用例:删掉的行复活 / 已确认写入丢失,两种方向都出现过)。 + // 更糟的是"旧世代排在最新记录之后":分片号复用会让同一序号对应两代记录, + // 恢复按序号排序时旧 INSERT 会排在一次 DELETE 之后被重放。 + // + // 现在分片号只增不减:删除只回收空间,不复用标识。 + if (obsolete.length === segments.length) { + const maxSeq = segments[segments.length - 1].seq; + const leftover = await this.listSegments(); + if (leftover.length === 0) { + this.currentSegment = Math.max(maxSeq + 1, keepFrom); + this.currentSize = 0; + this.segmentFirstLsn.clear(); + return keepFrom; + } + // 删除失败留下残片:它们都在 startSegment 之前(恢复时被忽略), + // 新记录必须继续用**更大的**分片号,避免与残片混进同一个文件。 + this.currentSegment = Math.max(leftover[leftover.length - 1].seq + 1, keepFrom); + this.currentSize = 0; + return keepFrom; + } + return keepFrom; } async exists(): Promise { diff --git a/tests/engine/aria-checksum.test.ts b/tests/engine/aria-checksum.test.ts index b9cde48..55fcae0 100644 --- a/tests/engine/aria-checksum.test.ts +++ b/tests/engine/aria-checksum.test.ts @@ -9,7 +9,7 @@ import { AriaEngine } from '../../src/engine/aria/index'; import { createSchema } from '../../src/table/schema'; -import { resetOPFSMock } from '../helpers/storage-harness'; +import { resetOPFSMock, readManifestNamespace } from '../helpers/storage-harness'; beforeEach(() => { resetOPFSMock(); }); @@ -41,12 +41,15 @@ async function listSSTKeys(engine: AriaEngine): Promise { return keys.filter((k) => k.startsWith('pg_')); } -/** 读取主 LSM 的 SSTable meta 列表 */ +/** + * 读取主 LSM 的 SSTable meta 列表(v0.8.0 B-6:改读 manifest)。 + * + * 元数据不再是独立的裸 JSON:它随 manifest 原子提交(带 CRC + 世代号)。 + * 旧布局下 `JSON.parse` 失败会被当成"没有文件",即元数据损坏 = 静默空库。 + */ async function listSSTMetas(engine: AriaEngine): Promise<{ id: number; pageIds?: number[] }[]> { const backend = (engine as any).backend; - const raw = await backend.read('__aria_lsm_meta'); - if (!raw) return []; - return JSON.parse(new TextDecoder().decode(raw)) as { id: number; pageIds?: number[] }[]; + return readManifestNamespace(backend, 'main'); } describe('AriaEngine — SSTable CRC 损坏检测(集成)', () => { diff --git a/tests/engine/aria-compression.test.ts b/tests/engine/aria-compression.test.ts index 1c9a39c..0ed7cff 100644 --- a/tests/engine/aria-compression.test.ts +++ b/tests/engine/aria-compression.test.ts @@ -58,7 +58,17 @@ async function writeAndMeasure(opts: { await engine.createTable(SCHEMA() as never); for (let i = 0; i < rows; i++) await engine.insert('t', [{ id: `k${i}`, blob: COMPRESSIBLE }]); - const lsm = (engine as unknown as { lsm: { sstableStore: { listMeta(): Promise> } } }).lsm; + const lsm = (engine as unknown as { + lsm: { flush(): Promise; sstableStore: { listMeta(): Promise> } }; + }).lsm; + // v0.8.0(B-6):**测量前显式 flush**。 + // + // 此前这里直接读 meta 求和,于是"落盘字节数"取决于测量瞬间有多少数据恰好在 + // SSTable 里 —— 而 manifest 提交(单一提交点)让每次 flush 多一次原子提交, + // 后台 flush 的进度随之变化,两次实验的"已落盘比例"不再相同,比值就变成在 + // 测时序而不是测压缩(实测:未 flush 时 off=19040/on=5130,flush 后 + // off=58570/on=10096 —— 后者才是同一份数据的真实压缩率)。 + await lsm.flush(); const metas = await lsm.sstableStore.listMeta(); const storedBytes = metas.reduce((sum, m) => sum + (m.totalSize ?? 0), 0); diff --git a/tests/engine/aria.test.ts b/tests/engine/aria.test.ts index 936a97a..93987a3 100644 --- a/tests/engine/aria.test.ts +++ b/tests/engine/aria.test.ts @@ -9,7 +9,7 @@ import { AriaEngine } from '../../src/engine/aria/index'; import { createSchema } from '../../src/table/schema'; import { MetonaSqlark } from '../../src/core'; -import { resetOPFSMock } from '../helpers/storage-harness'; +import { resetOPFSMock, readManifestState } from '../helpers/storage-harness'; beforeEach(() => { resetOPFSMock(); }); @@ -455,13 +455,14 @@ describe('AriaEngine — Schema 持久化 (Memory Backend)', () => { name: { type: 'string', required: true }, })); - // 直接通过 backend 验证 Schema JSON 已写入 - const raw = await (engine as any).backend.read('__aria_schemas'); - expect(raw).not.toBeNull(); - const json = new TextDecoder().decode(raw); - const data = JSON.parse(json); - expect(data.users).toBeDefined(); - expect(data.users.id.primaryKey).toBe(true); + // v0.8.0(B-6):表结构不再是独立的裸 JSON(__aria_schemas), + // 而是随 manifest 一起**原子提交**(单一提交点)。测试改读 manifest —— + // 它才是落盘结构的权威来源(带 CRC 与世代号)。 + // 旧布局的问题:坏 JSON 会被 `readMetaList()` 当成 `[]`,元数据损坏 = 静默空库。 + const manifest = await readManifestState((engine as any).backend); + expect(manifest).not.toBeNull(); + expect(manifest!.schemas.users).toBeDefined(); + expect(manifest!.schemas.users.id.primaryKey).toBe(true); await engine.close(); }); diff --git a/tests/helpers/storage-harness.ts b/tests/helpers/storage-harness.ts index b37d74d..b7900f7 100644 --- a/tests/helpers/storage-harness.ts +++ b/tests/helpers/storage-harness.ts @@ -518,3 +518,36 @@ export class CrashableStoreBackend implements IStorageBackend { this.store.commitAll(); } } + +// --------------------------------------------------------------------------- +// v0.8.0(B-6):manifest 读取辅助 +// --------------------------------------------------------------------------- + +/** + * 读取存储上**最新有效世代**的 manifest(用生产解码器,不做测试特供路径)。 + * + * 为什么测试需要它:v0.8.0 起 SSTable 元数据与表结构不再是各自独立的裸 JSON + * (`__aria_lsm_meta*` / `__aria_schemas`),而是随 manifest 一起原子提交。 + * 任何"直接读裸 JSON 验证落盘内容"的测试都必须改读 manifest —— 否则它验证的是 + * 一个已经不存在(且没有 CRC/世代保护)的存储布局。 + * + * @returns manifest 内容;全新库(没有任何 manifest 文件)返回 null; + * 文件存在但全部世代无效 → 抛错(与生产恢复语义一致,绝不"失败当空库") + */ +export async function readManifestState( + backend: IStorageBackend, +): Promise { + const { ManifestStore } = await import('../../src/engine/aria/store/manifest'); + const store = new ManifestStore({ backend }); + const loaded = await store.load(); + return loaded.manifest; +} + +/** 读取某个命名空间当前的 SSTable meta 列表(测试断言用) */ +export async function readManifestNamespace( + backend: IStorageBackend, + ns: string = 'main', +): Promise<{ id: number; level: number; pageIds?: number[]; minKey: string; maxKey: string }[]> { + const manifest = await readManifestState(backend); + return manifest?.namespaces[ns]?.sstables ?? []; +} diff --git a/tests/v042-fixes.test.ts b/tests/v042-fixes.test.ts index dbf0037..32be055 100644 --- a/tests/v042-fixes.test.ts +++ b/tests/v042-fixes.test.ts @@ -20,7 +20,7 @@ import { OPFSBackend } from '../src/engine/aria/store/opfs_backend'; import { KVStoreEngine } from '../src/engine/kvstore_engine'; import type { SSTableMeta } from '../src/engine/aria/types'; -import { resetOPFSMock } from './helpers/storage-harness'; +import { resetOPFSMock, readManifestNamespace } from './helpers/storage-harness'; beforeEach(() => { resetOPFSMock(); }); @@ -153,8 +153,8 @@ describe('P0-1b — AriaEngine 打开时完整性校验', () => { // 篡改存储:把第一个 SSTable 的第一个页面写成残缺内容(meta 仍引用它) const backend = new OPFSBackend(); await backend.open(dbName); - const metas = JSON.parse(new TextDecoder().decode( - await backend.read('__aria_lsm_meta') as ArrayBuffer)) as { id: number; pageIds: number[] }[]; + // v0.8.0(B-6):SSTable meta 随 manifest 原子提交(不再是裸 JSON key) + const metas = await readManifestNamespace(backend, 'main') as { id: number; pageIds: number[] }[]; expect(metas.length).toBeGreaterThan(0); const pageId = metas[0].pageIds[0]; await backend.write(`pg_${pageId}`, new TextEncoder().encode('truncated-garbage').buffer); @@ -174,8 +174,7 @@ describe('P0-1b — AriaEngine 打开时完整性校验', () => { const backend2 = new OPFSBackend(); await backend2.open(dbName); const remaining = (await backend2.listKeys()).filter((k) => k.startsWith('pg_')); - const metaRaw = await backend2.read('__aria_lsm_meta'); - const metaList = JSON.parse(new TextDecoder().decode(metaRaw ?? new Uint8Array())) as { pageIds?: number[] }[]; + const metaList = await readManifestNamespace(backend2, 'main'); const livePages = new Set(); for (const m of metaList) if (m.pageIds) for (const pid of m.pageIds) livePages.add(pid); const orphan = remaining.filter((k) => !livePages.has(Number(k.slice(3)))); @@ -414,8 +413,7 @@ describe('P2-9 — 统一自愈接口 repair / clearAll', () => { // 篡改一个 SSTable 的页面文件 const backend = new OPFSBackend(); await backend.open(dbName); - const metas = JSON.parse(new TextDecoder().decode( - await backend.read('__aria_lsm_meta') as ArrayBuffer)) as { id: number; pageIds: number[] }[]; + const metas = await readManifestNamespace(backend, 'main') as { id: number; pageIds: number[] }[]; expect(metas.length).toBeGreaterThan(0); const victimPage = metas[0].pageIds[0]; const raw = new Uint8Array(await backend.read(`pg_${victimPage}`) as ArrayBuffer); diff --git a/tests/v042-hardening.test.ts b/tests/v042-hardening.test.ts index cebd2f1..6b01c56 100644 --- a/tests/v042-hardening.test.ts +++ b/tests/v042-hardening.test.ts @@ -116,7 +116,8 @@ describe('P1-A — 二级索引恢复', () => { const idxLsm = (engine2 as any).secondaryIndexes.get('users:idx:email'); expect(idxLsm).toBeDefined(); expect(idxLsm.getStats().sstableCount).toBeGreaterThan(0); - await idxLsm.prefetchRange('', '\uffff'); + // v0.8.0(B-6/55):`prefetchRange` 已删除 —— 读取自洽(未命中即回源 + CRC + // 校验),调用方不再需要"先预加载再读"这条隐式约定。 // v0.8.0: LSM.rangeScan 改为 async(读取自洽,未命中会回源加载) expect(await idxLsm.rangeScan('', '\uffff')).toHaveLength(3); const byEmail = await engine2.find('users', { table: 'users', where: { email: 'a@x.com' } }); @@ -147,8 +148,7 @@ describe('P1-A — 二级索引恢复', () => { // 强断言:索引 LSM 已恢复且包含 WAL 回放的行(崩溃前索引未更新,恢复后必须重建) const idxLsm = (engine2 as any).secondaryIndexes.get('users:idx:city'); expect(idxLsm).toBeDefined(); - await idxLsm.prefetchRange('', '\uffff'); - // v0.8.0: LSM.rangeScan 改为 async + // v0.8.0(B-6/55):prefetchRange 已删除(读取自洽);rangeScan 现为 async expect(await idxLsm.rangeScan('', '\uffff')).toHaveLength(2); // 索引查询应看到 WAL 恢复的行(修复前索引与主数据不一致 → 丢行) const byCity = await engine2.find('users', { table: 'users', where: { city: 'Shanghai' } }); diff --git a/tests/v044-hardening.test.ts b/tests/v044-hardening.test.ts index 4cd6368..f9ed2f4 100644 --- a/tests/v044-hardening.test.ts +++ b/tests/v044-hardening.test.ts @@ -120,8 +120,20 @@ describe('P0 — flush 报告后台失败', () => { await new Promise((r) => setTimeout(r, 50)); // flush 必须报告后台失败(修复前静默吞错) await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' }); - // 再次 flush:错误已消费,正常完成 - await expect(lsm.flush()).resolves.toBeUndefined(); + + // v0.8.0(B-6/44+45)契约变更:第二次 flush **同样必须失败**。 + // + // 修复前的第二行断言是 `resolves.toBeUndefined()` —— 它编码的语义是 + // "错误已消费 → 这次 flush 算成功"。但 `FailingStore.save` 是**永远**失败, + // 那份数据此时仍然只在内存里(WAL 之外没有任何副本):报告"成功"等于告诉 + // 调用方"已落盘",而崩溃就会丢。这属于"静默成功",与本次根治的目标正好相反。 + // + // 现在的语义:flush 只有在**真的把数据落盘**后才 resolve;失败可以重试 + // (冻结表会被重新入链,见下一条用例),但重试仍失败就必须继续报错。 + await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' }); + // 冻结表仍在(可读、可重试),数据没有凭空消失 + expect(lsm.getStats().frozenTables).toBeGreaterThan(0); + expect(await lsm.get('k0')).toEqual({ v: 0 }); }); it('后台 compaction 失败后 flush() 报告(链不卡死)', async () => { diff --git a/tests/v080-aria-ddl-atomicity.test.ts b/tests/v080-aria-ddl-atomicity.test.ts index 8eff8ad..deb81c6 100644 --- a/tests/v080-aria-ddl-atomicity.test.ts +++ b/tests/v080-aria-ddl-atomicity.test.ts @@ -29,6 +29,13 @@ import { describe, it, expect, beforeEach } from '@jest/globals'; import { AriaEngine } from '../src/engine/aria/index'; import { resetOPFSMock } from './helpers/storage-harness'; import type { IStorageBackend } from '../src/engine/aria/store/backend'; +import { + decodeManifest, + encodeManifest, + generationFromKey, + manifestKey, + type AriaManifest, +} from '../src/engine/aria/store/manifest'; beforeEach(() => { resetOPFSMock(); }); @@ -71,16 +78,71 @@ class OrderRecordingBackend implements IStorageBackend { async exists(k: string): Promise { return this.files.has(k); } async clear(): Promise { this.files.clear(); } - /** 索引:WAL 记录写入发生在 schema 落盘**之前** */ + /** + * 索引:WAL 记录写入发生在 schema **持久化之前**。 + * + * v0.8.0(B-6):schema 不再是独立的 `__aria_schemas` 裸 JSON, + * 它随 manifest 原子提交 —— 因此"schema 落盘"的落点变成 `__aria_manifest_*`。 + * 断言的**性质**没变(WAL 先于持久化),只是落点换了。 + */ walPrecedesSchema(): boolean { const wal = this.order.findIndex((k) => k.includes('wal')); - const sch = this.order.findIndex((k) => k.includes('schemas')); + const sch = this.order.findIndex((k) => k.includes('__aria_manifest_')); return wal >= 0 && sch >= 0 && wal < sch; } /** 模拟"该文件没能落盘"(崩溃窗口) */ dropFile(pattern: string): void { for (const k of [...this.files.keys()]) if (k.includes(pattern)) this.files.delete(k); } + /** + * 当前 manifest 的**语义快照**(解码后的内容 + 世代号)。 + * 用于"DDL 之前"的状态存档:见 `rollbackManifest()`。 + */ + snapshotManifest(): { manifest: AriaManifest; generation: number } { + const gens = [...this.files.keys()] + .map((k) => generationFromKey(k)) + .filter((g): g is number => g !== null) + .sort((a, b) => b - a); + if (gens.length === 0) throw new Error('no manifest on medium'); + const raw = this.files.get(manifestKey(gens[0]))!; + const decoded = decodeManifest(new Uint8Array(raw)); + if (!decoded.ok) throw new Error(`manifest decode failed: ${decoded.reason}`); + return { manifest: decoded.manifest, generation: gens[0] }; + } + + /** + * 把 manifest 回滚到给定快照(写成一份**更新世代号**的内容,并清掉其它世代)。 + * + * 语义:模拟"这次 DDL 的 manifest 提交从未落盘"——数据文件与 WAL 保持原样, + * 于是重开时结构只能靠 **WAL 意图回放** 恢复(这正是本套件要验证的兜底)。 + * + * 注意:不能简单删掉新世代 —— 世代保留窗口是 2,被回滚到的那一代此时 + * 可能已经被正常清理掉了。因此这里用快照内容重新提交一份新世代。 + */ + rollbackManifest(snapshot: { manifest: AriaManifest; generation: number }): void { + const gens = [...this.files.keys()] + .map((k) => generationFromKey(k)) + .filter((g): g is number => g !== null); + const nextGeneration = Math.max(0, ...gens) + 1; + const rolled: AriaManifest = { + ...snapshot.manifest, + generation: nextGeneration, + namespaces: snapshot.manifest.namespaces, + schemas: snapshot.manifest.schemas, + wal: { ...snapshot.manifest.wal }, + frozen: [], + owner: { ...snapshot.manifest.owner }, + }; + const bytes = encodeManifest(rolled); + this.files.set( + manifestKey(nextGeneration), + bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer, + ); + for (const k of [...this.files.keys()]) { + const gen = generationFromKey(k); + if (gen !== null && gen !== nextGeneration) this.files.delete(k); + } + } } /** @@ -148,13 +210,15 @@ describe('[v0.8.0] A41 DDL 结构变更可崩溃恢复(WAL 兜底)', () => { const backend = new OrderRecordingBackend(); const engine = openWith(backend); await engine.open('ddl-alter-crash', 1); + // 记录"DDL 之前"的 manifest 状态:崩溃窗口 = 之后的提交都没落盘 + const beforeDdl = backend.snapshotManifest(); await engine.createTable(schema('t')); await engine.alterTable('t', 'ADD', { name: 'tag', type: 'string' } as never); await engine.alterTable('t', 'ADD', { name: 'tag2', type: 'string' } as never); // 崩溃窗口:两次 ALTER 的 schema 都没能落盘(不 close,否则 close 会重试落盘) - backend.dropFile('__aria_schemas'); - expect((await backend.listKeys()).some((k) => k.includes('schemas'))).toBe(false); + backend.rollbackManifest(beforeDdl); + expect(backend.snapshotManifest().generation).toBeGreaterThan(beforeDdl.generation); const engine2 = openWith(backend); await engine2.open('ddl-alter-crash', 1); @@ -173,9 +237,10 @@ describe('[v0.8.0] A41 DDL 结构变更可崩溃恢复(WAL 兜底)', () => { await engine.insert('t', [{ id: 'r1' }]); await engine.createTable(schema('other')); await engine.insert('other', [{ id: 'o1' }]); + const beforeDrop = backend.snapshotManifest(); await engine.dropTable('other'); - backend.dropFile('__aria_schemas'); // schema 落盘丢失 + backend.rollbackManifest(beforeDrop); // DROP 的持久化提交丢失 // 不 close(崩溃语义) const engine2 = openWith(backend); @@ -191,9 +256,10 @@ describe('[v0.8.0] A41 DDL 结构变更可崩溃恢复(WAL 兜底)', () => { const backend = new OrderRecordingBackend(); const engine = openWith(backend); await engine.open('ddl-create-crash', 1); + const beforeCreate = backend.snapshotManifest(); await engine.createTable(schema('t')); await engine.insert('t', [{ id: 'r1' }]); - backend.dropFile('__aria_schemas'); + backend.rollbackManifest(beforeCreate); const engine2 = openWith(backend); await engine2.open('ddl-create-crash', 1); diff --git a/tests/v080-b6-single-commit-point.test.ts b/tests/v080-b6-single-commit-point.test.ts new file mode 100644 index 0000000..8061c3f --- /dev/null +++ b/tests/v080-b6-single-commit-point.test.ts @@ -0,0 +1,1404 @@ +/** + * v0.8.0(B-6)回归套件 —— 存储层**单一提交点**(`__aria_manifest`)与 LSM 结构根治 + * ============================================================================ + * 本套件覆盖 PLAN-v0.7.5.md 工作流 B 的 B-6 全部条目。每一条都对应一个**实测过** + * 的缺陷(或一条被写进 manifest 的持久不变量): + * + * | 编号 | 缺陷 / 不变量 | 本文件对应用例 | + * |---|---|---| + * | B-6 核心 | 元数据各自独立落盘 → 崩溃窗口内互相矛盾 | `manifest 单一提交点` | + * | 消除 7 | SSTable meta 损坏 → 静默空库 + repair 删活页 | `元数据损坏必须显式失败` | + * | 消除 4 | 介质读故障被折叠为"文件不存在" → meta 被误删 | `介质读故障 ≠ 文件缺失` | + * | 消除 9 | WAL 分片空洞 → 尾部静默丢弃 | `WAL 分片空洞必须显式失败` | + * | 消除 5 | 陈旧实例覆盖新实例的 meta | `陈旧实例提交被拒绝` | + * | 44 | 后台 flush 失败后冻结表无重试路径 → 崩溃即丢 | `flush 失败可重试` | + * | 45 | 后台错误检查在入链之前 → 本次 flush 被整个跳过 | `flush 必须先入链再报错` | + * | 47 | 提前终止时底层生成器多产出 1 条 | `流式扫描提前终止不多算` | + * | 49 | `compacting` 单 boolean → 跨层触发被静默丢弃 | `按层 compaction 状态` | + * | 50 | compaction 先 `splice` 整层 → 窗口内该层对读者不可见 | `compaction 期间读不到空层` | + * | 51 | 墓碑/历史版本永不回收 | `底部层合并回收墓碑` | + * | 55 | checkpoint 等完整 compaction → 写路径秒级卡顿 | `checkpoint 不等 compaction` | + * | 冻结意图 | "已确认写入"是否真的还在,此前不可观测 | `冻结意图阻止水位推进` | + * + * 变异验证:把对应实现回退到修复前的行为,这些用例必须失败(见各用例注释)。 + */ +import { describe, it, expect, beforeEach } from '@jest/globals'; +import { AriaEngine } from '../src/engine/aria/index'; +import { LSM } from '../src/engine/aria/index/lsm'; +import { MergeIterator, type EntrySource } from '../src/engine/aria/index/merge_iterator'; +import { SSTableReader } from '../src/engine/aria/index/sstable'; +import { + ManifestStore, + MANIFEST_HEADER_SIZE, + createEmptyManifest, + decodeManifest, + encodeManifest, + generationFromKey, + manifestKey, + type AriaManifest, +} from '../src/engine/aria/store/manifest'; +import { MemoryBackend, type IStorageBackend } from '../src/engine/aria/store/backend'; +import { crc32 } from '../src/engine/aria/crc32'; +import { OPFSBackend } from '../src/engine/aria/store/opfs_backend'; +import { SegmentedWALStore } from '../src/engine/aria/wal/segmented_store'; +import { WAL } from '../src/engine/aria/wal/log'; +import { WALRecordType, MAX_LSM_LEVELS } from '../src/engine/aria/types'; +import { resetOPFSMock, readManifestState } from './helpers/storage-harness'; +import { createSchema } from '../src/table/schema'; + +jest.setTimeout(60000); + +beforeEach(() => { resetOPFSMock(); }); + +let dbCounter = 0; +function uniqueDB(tag: string): string { + return `${tag}-${Date.now()}-${++dbCounter}`; +} + +const SCHEMA = () => createSchema('t', { + id: { type: 'string', primaryKey: true }, + v: { type: 'number' }, +}); + +function rows(n: number): Record[] { + const out: Record[] = []; + for (let i = 0; i < n; i++) out.push({ id: `k${i}`, v: i }); + return out; +} + +// --------------------------------------------------------------------------- +// 可注入故障/门控的存储后端(包住真实后端,不替换语义) +// --------------------------------------------------------------------------- + +class GatedBackend implements IStorageBackend { + /** 命中门控时置位(测试用它等待"引擎已经走到这个点") */ + onGated: (() => void) | null = null; + private gate: Promise = Promise.resolve(); + private releaseGate: (() => void) | null = null; + /** 需要门控的 key 前缀(空 = 不门控) */ + gatePattern: string | null = null; + /** 门控前放行的匹配次数(默认 0 = 第一次匹配就阻塞) */ + gateSkip = 0; + /** 这些 key 前缀上的读会抛错(模拟介质故障) */ + failReadPattern: string | null = null; + /** 这些 key 前缀上的写会抛错 */ + failWritePattern: string | null = null; + + constructor(private inner: IStorageBackend) {} + + /** 让第 `skip + 1` 次匹配的写阻塞,直到 `release()` */ + armGate(pattern: string, skip = 0): void { + this.gatePattern = pattern; + this.gateSkip = skip; + this.gate = new Promise((resolve) => { this.releaseGate = resolve; }); + } + + release(): void { + this.gatePattern = null; + if (this.releaseGate) { this.releaseGate(); this.releaseGate = null; } + } + + private matches(pattern: string | null, key: string): boolean { + return pattern !== null && key.startsWith(pattern); + } + + open(name: string): Promise { return this.inner.open(name); } + close(): Promise { return this.inner.close(); } + isOpen(): boolean { return this.inner.isOpen(); } + + async read(key: string): Promise { + if (this.matches(this.failReadPattern, key)) { + throw new Error(`injected read failure on "${key}"`); + } + return this.inner.read(key); + } + + async write(key: string, data: ArrayBuffer): Promise { + if (this.matches(this.failWritePattern, key)) { + throw new Error(`injected write failure on "${key}"`); + } + if (this.matches(this.gatePattern, key)) { + if (this.gateSkip > 0) { + this.gateSkip--; + } else { + // 一次性门控:只阻塞这一次匹配的写;后续写必须能继续(否则测试测到的 + // 是"整个介质被冻住",而不是"某一次 compaction 卡住") + this.gatePattern = null; + const notify = this.onGated; + if (notify) { this.onGated = null; notify(); } + await this.gate; + } + } + return this.inner.write(key, data); + } + + append(key: string, data: ArrayBuffer): Promise { + if (this.matches(this.failWritePattern, key)) { + return Promise.reject(new Error(`injected append failure on "${key}"`)); + } + return this.inner.append ? this.inner.append(key, data) : this.inner.write(key, data); + } + + writeMany(entries: Record): Promise { + const keys = Object.keys(entries); + if (this.failWritePattern && keys.some((k) => this.matches(this.failWritePattern, k))) { + return Promise.reject(new Error('injected writeMany failure')); + } + return this.inner.writeMany(entries); + } + + delete(key: string): Promise { return this.inner.delete(key); } + deleteMany(keys: string[]): Promise { return this.inner.deleteMany(keys); } + listKeys(): Promise { return this.inner.listKeys(); } + exists(key: string): Promise { return this.inner.exists(key); } + clear(): Promise { return this.inner.clear(); } +} + +/** 一次写失败后自动恢复正常(模拟"瞬时故障",用于验证重试路径) */ +class FailOnceBackend implements IStorageBackend { + remainingFailures = 1; + /** 默认匹配"整 value SSTable"(memory 后端不页面化);页面化时传 'pg_' */ + failPattern = 'sst_'; + constructor(private inner: IStorageBackend) {} + open(name: string): Promise { return this.inner.open(name); } + close(): Promise { return this.inner.close(); } + isOpen(): boolean { return this.inner.isOpen(); } + read(key: string): Promise { return this.inner.read(key); } + async write(key: string, data: ArrayBuffer): Promise { + if (this.remainingFailures > 0 && key.startsWith(this.failPattern)) { + this.remainingFailures--; + throw new Error(`injected ONE write failure on "${key}"`); + } + return this.inner.write(key, data); + } + append(key: string, data: ArrayBuffer): Promise { + return this.inner.append ? this.inner.append(key, data) : this.inner.write(key, data); + } + writeMany(entries: Record): Promise { + for (const [k] of Object.entries(entries)) { + if (this.remainingFailures > 0 && k.startsWith(this.failPattern)) { + this.remainingFailures--; + return Promise.reject(new Error(`injected ONE write failure on "${k}"`)); + } + } + return this.inner.writeMany(entries); + } + delete(key: string): Promise { return this.inner.delete(key); } + deleteMany(keys: string[]): Promise { return this.inner.deleteMany(keys); } + listKeys(): Promise { return this.inner.listKeys(); } + exists(key: string): Promise { return this.inner.exists(key); } + clear(): Promise { return this.inner.clear(); } +} + +function openEngine( + dbName: string, + backend: IStorageBackend, + extra: Record = {}, +): AriaEngine { + return new AriaEngine({ + storageBackend: 'memory', + checkpointInterval: 100_000_000, + memtableSizeThreshold: 64 * 1024 * 1024, + testBackend: backend, + ...extra, + } as never); +} + +// =========================================================================== +// 1. ManifestStore 单元语义(真实现,不做替身) +// =========================================================================== + +describe('[v0.8.0][B-6] manifest 单一提交点', () => { + function newStore(instanceId = 'inst-A') { + const backend = new MemoryBackend(); + const store = new ManifestStore({ backend, instanceId, now: () => 1000 }); + return { backend, store }; + } + + it('提交 → 重新 load 得到同一内容,世代号单调递增', async () => { + const { backend, store } = newStore(); + await store.load(); + store.current.schemas = { t: { id: { type: 'string', primaryKey: true } } as never }; + const first = await store.commit(); + expect(first.generation).toBe(1); + expect(first.owner.instanceId).toBe('inst-A'); + expect(first.owner.epoch).toBe(1); + + store.current.pageIdWatermark = 42; + const second = await store.commit(); + expect(second.generation).toBe(2); + + const reloaded = await new ManifestStore({ backend }).load(); + expect(reloaded.generation).toBe(2); + expect(reloaded.manifest!.pageIdWatermark).toBe(42); + expect(reloaded.manifest!.schemas.t.id.primaryKey).toBe(true); + expect(reloaded.manifest!.owner.instanceId).toBe('inst-A'); + }); + + it('payload 被篡改 → 该世代无效,回退到上一代(并记录损坏世代)', async () => { + const { backend, store } = newStore(); + await store.load(); + store.current.schemas = { t: { id: { type: 'string' } } as never }; + await store.commit(); + store.current.schemas = { t: { id: { type: 'string' } }, u: { id: { type: 'string' } } as never }; + const newest = await store.commit(); + expect(newest.generation).toBe(2); + store.current.schemas = { t: { id: { type: 'string' } }, u: { id: { type: 'string' } } as never, x: {} as never }; + await store.commit(); // gen 3 + + // 篡改最新世代的载荷(保持头部 CRC 有效 → 由载荷 CRC 拦住) + const key = manifestKey(3); + const bytes = new Uint8Array((await backend.read(key))!); + bytes[MANIFEST_HEADER_SIZE + 3] ^= 0xff; + await backend.write(key, bytes.buffer as ArrayBuffer); + + const loaded = await new ManifestStore({ backend }).load(); + expect(loaded.generation).toBe(2); // 回退到上一代 + expect(loaded.hadInvalidGenerations).toBe(true); + expect(loaded.skipped.map((s) => s.generation)).toEqual([3]); + expect((loaded.manifest!.schemas as Record).x).toBeUndefined(); + }); + + it('全部世代都损坏 → 抛 ARIA_MANIFEST_CORRUPT(绝不返回空状态)', async () => { + const { backend, store } = newStore(); + await store.load(); + store.current.schemas = { t: { id: { type: 'string' } } as never }; + await store.commit(); + + const key = manifestKey(1); + const bytes = new Uint8Array((await backend.read(key))!); + bytes.fill(0, MANIFEST_HEADER_SIZE); // 抹掉载荷 + await backend.write(key, bytes.buffer as ArrayBuffer); + + // 变异验证:若把 load() 的"全部无效"改回返回 null(旧行为 = 空库), + // 这条断言立即失败 —— 而"静默空库"正是审计里的 P0 表现。 + await expect(new ManifestStore({ backend }).load()).rejects.toMatchObject({ + code: 'ARIA_MANIFEST_CORRUPT', + }); + }); + + it('头部自带 CRC:世代号/长度被篡改的世代一律判无效', async () => { + const { store } = newStore(); + await store.load(); + await store.commit(); + const bytes = encodeManifest({ ...store.current, generation: 1 }); + // 篡改头部世代号(不改 header CRC)→ 头部 CRC 校验失败 + const tampered = new Uint8Array(bytes); + new DataView(tampered.buffer).setUint32(8, 99, false); + const decoded = decodeManifest(tampered); + expect(decoded.ok).toBe(false); + if (!decoded.ok) expect(decoded.reason).toContain('header CRC mismatch'); + }); + + it('陈旧实例提交被拒绝(STALE_INSTANCE),不会静默覆盖新世代', async () => { + const backend = new MemoryBackend(); + const a = new ManifestStore({ backend, instanceId: 'A' }); + const b = new ManifestStore({ backend, instanceId: 'B' }); + await a.load(); + await b.load(); // 两个实例都看到 generation = 0 + + b.current.schemas = { fromB: {} as never }; + const bCommitted = await b.commit(); + expect(bCommitted.generation).toBe(1); + + // A 的内存态已经落后(B 提交过)→ A 的提交必须被拒绝,而不是拿陈旧状态覆盖 + a.current.schemas = { staleA: {} as never }; + await expect(a.commit()).rejects.toMatchObject({ code: 'STALE_INSTANCE' }); + + // B 的内容仍然在(陈旧实例没能覆盖) + const loaded = await new ManifestStore({ backend }).load(); + expect(Object.keys(loaded.manifest!.schemas)).toEqual(['fromB']); + expect(loaded.manifest!.owner.instanceId).toBe('B'); + }); + + it('提交是"先写后验":回读不可用 → ARIA_MANIFEST_WRITE_FAILED,内存态不前进', async () => { + const inner = new MemoryBackend(); + const backend = new (class extends MemoryBackend { + hideReads = false; + async read(key: string): Promise { + if (this.hideReads && key.startsWith('__aria_manifest_')) return null; + return super.read(key); + } + })(); + await backend.open('verify-fail'); + const store = new ManifestStore({ backend: backend as unknown as IStorageBackend, instanceId: 'A' }); + await store.load(); + backend.hideReads = true; + await expect(store.commit()).rejects.toMatchObject({ code: 'ARIA_MANIFEST_WRITE_FAILED' }); + expect(store.currentGeneration).toBe(0); // 没有假装提交成功 + void inner; + }); + + it('保留至少两代;本轮没有损坏世代时才清理更早的世代', async () => { + const { backend, store } = newStore(); + await store.load(); + for (let i = 0; i < 5; i++) await store.commit(); + let gens = (await backend.listKeys()).map((k) => generationFromKey(k)).filter((g) => g !== null); + expect(new Set(gens)).toEqual(new Set([4, 5])); + + // 造一个"更新但损坏"的世代(gen 6)→ 加载时会跳过它并记录 + const bogus = createEmptyManifest({ instanceId: 'bogus' }); + const broken = new Uint8Array(encodeManifest({ ...bogus, generation: 6 })); + broken.fill(0, MANIFEST_HEADER_SIZE); + await backend.write(manifestKey(6), broken.buffer as ArrayBuffer); + + const store2 = new ManifestStore({ backend, instanceId: 'A' }); + const loaded = await store2.load(); + expect(loaded.generation).toBe(5); // 回退到最新有效世代 + expect(loaded.hadInvalidGenerations).toBe(true); // 并记录损坏世代 + + await store2.commit(); // → gen 7 + gens = (await backend.listKeys()).map((k) => generationFromKey(k)).filter((g) => g !== null); + // 存在损坏世代时不做任何清理("引用不到的东西一律保留而非删除") + expect(gens).toContain(4); + expect(gens).toContain(5); + expect(gens).toContain(6); + }); +}); + +// =========================================================================== +// 2. 引擎级:数据 → manifest → WAL 截断 +// =========================================================================== + +describe('[v0.8.0][B-6] 引擎的提交顺序与旧格式迁移', () => { + it('flush 后:SSTable 元数据在 manifest 里,且 WAL 水位只在水位覆盖后才推进', async () => { + const backend = new MemoryBackend(); + await backend.open('b6-order'); + // 显式开启页面化:这样 meta 里会带 pageIds(页面化是 v0.8.0 的默认路径) + const engine = openEngine('b6-order', backend, { memtableSizeThreshold: 2048, pageStorage: true }); + await engine.open('b6-order', 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(40)); + + const before = await readManifestState(backend); + expect(before).not.toBeNull(); + // 未显式 flush 时,数据可能还在 memtable → 水位不得推进到"最新 LSN" + const walLsnBefore = (engine as any).wal.getLsn() as number; + expect(before!.wal.startLsn).toBeLessThanOrEqual(walLsnBefore); + + await (engine as any).lsm.flush(); + const after = await readManifestState(backend); + expect(after!.namespaces.main.sstables.length).toBeGreaterThan(0); + expect(after!.wal.startLsn).toBeGreaterThanOrEqual(before!.wal.startLsn); // 单调 + expect(after!.wal.nextLsn).toBeGreaterThanOrEqual(after!.wal.startLsn); + // meta 里必须带页面映射(页面化路径的数据定位依据) + expect(after!.namespaces.main.sstables[0].pageIds?.length ?? 0).toBeGreaterThan(0); + await engine.close(); + }); + + it('旧格式(__aria_lsm_meta/__aria_schemas)库打开后完整迁移,旧键保留', async () => { + const dbName = uniqueDB('b6-legacy'); + const engine = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024, checkpointInterval: 100_000_000 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(20)); + await (engine as any).lsm.flush(); + await engine.close(); + + // 把 v0.8.0 布局"降级"成旧布局:写回裸 JSON meta / schema,删掉 manifest + const backend = new OPFSBackend(); + await backend.open(dbName); + const manifest = (await readManifestState(backend))!; + const mainMetas = manifest.namespaces.main.sstables; + await backend.write('__aria_lsm_meta', new TextEncoder().encode(JSON.stringify(mainMetas)).buffer); + await backend.write('__aria_schemas', new TextEncoder().encode(JSON.stringify(manifest.schemas)).buffer); + const pageMeta = new ArrayBuffer(8); + new DataView(pageMeta).setUint32(0, manifest.pageIdWatermark, false); + await backend.write('__aria_meta', pageMeta); + await backend.deleteMany( + (await backend.listKeys()).filter((k) => k.startsWith('__aria_manifest_')), + ); + await backend.close(); + + const engine2 = new AriaEngine({ storageBackend: 'opfs', checkpointInterval: 100_000_000 }); + await engine2.open(dbName, 1); + expect(engine2.getRecoveryReport().legacyImported).toBe(true); + expect(await engine2.count('t')).toBe(20); + await engine2.close(); + + const backend2 = new OPFSBackend(); + await backend2.open(dbName); + const migrated = await readManifestState(backend2); + expect(migrated).not.toBeNull(); + expect(migrated!.namespaces.main.sstables.length).toBe(mainMetas.length); + expect(Object.keys(migrated!.schemas)).toEqual(['t']); + // 旧键**保留**("引用不到的东西一律保留而非删除") + expect(await backend2.exists('__aria_lsm_meta')).toBe(true); + await backend2.close(); + }); + + it('旧格式 meta 损坏 → 抛 ARIA_LEGACY_META_CORRUPT(修复前是静默空库)', async () => { + const dbName = uniqueDB('b6-legacy-corrupt'); + const backend = new OPFSBackend(); + await backend.open(dbName); + await backend.write('__aria_lsm_meta', new TextEncoder().encode('{not json').buffer); + await backend.close(); + + const engine = new AriaEngine({ storageBackend: 'opfs' }); + await expect(engine.open(dbName, 1)).rejects.toMatchObject({ code: 'ARIA_LEGACY_META_CORRUPT' }); + }); + + it('最新世代损坏 → 回退上一代并标记 manifestFallback,数据仍可读', async () => { + const dbName = uniqueDB('b6-fallback'); + const engine = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024, checkpointInterval: 100_000_000 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(15)); + await engine.close(); + + const backend = new OPFSBackend(); + await backend.open(dbName); + const gens = (await backend.listKeys()) + .map((k) => generationFromKey(k)).filter((g): g is number => g !== null).sort((a, b) => b - a); + const newest = new Uint8Array((await backend.read(manifestKey(gens[0])))!); + newest.fill(0, MANIFEST_HEADER_SIZE); + await backend.write(manifestKey(gens[0]), newest.buffer as ArrayBuffer); + await backend.close(); + + const engine2 = new AriaEngine({ storageBackend: 'opfs', checkpointInterval: 100_000_000 }); + await engine2.open(dbName, 1); + expect(engine2.getRecoveryReport().manifestFallback).toBe(true); + expect(await engine2.count('t')).toBe(15); + await engine2.close(); + }); + + it('介质读故障 ≠ 文件缺失:读取报 ARIA_SSTABLE_READ_FAILED,meta 不被自愈删除', async () => { + const dbName = uniqueDB('b6-readfail'); + const base = new MemoryBackend(); + // 先正常写入并落盘(写入路径不带故障注入) + const writer = openEngine(dbName, base, { memtableSizeThreshold: 2048 }); + await writer.open(dbName, 1); + await writer.createTable(SCHEMA()); + await writer.insert('t', rows(30)); + await (writer as any).lsm.flush(); + // 注意:**不能** close —— MemoryBackend.close() 会清空介质(等价于删库), + // 这里要的是"同一个介质上换一个引擎实例"。 + + // 新引擎(缓存为空 → 读取必然回源)打开时会重放 WAL 并再落一次盘, + // 因此 meta 基线要在打开**之后**取 + const gated = new GatedBackend(base); + const reader = openEngine(dbName, gated, { memtableSizeThreshold: 2048 }); + await reader.open(dbName, 1); + const metasBefore = (await readManifestState(base))!.namespaces.main.sstables.length; + expect(metasBefore).toBeGreaterThan(0); + expect(await reader.count('t')).toBe(30); // 此时还能读(缓存已建立) + gated.failReadPattern = 'sst_'; // 让后续回源读失败 + (reader as any).lsm.sstableCache.clear(); // 清缓存 → 强制回源 + (reader as any).lsm.cacheSize = 0; + (reader as any).lsm.oversizedSSTables.clear(); + + await expect(reader.find('t', { table: 't' })).rejects.toMatchObject({ + code: 'ARIA_SSTABLE_READ_FAILED', + }); + + // 修复前的行为:读故障被当成"文件不存在" → dropInvalidSSTable 删掉 meta(不可逆) + const metasAfter = (await readManifestState(base))!.namespaces.main.sstables.length; + expect(metasAfter).toBe(metasBefore); + + // 故障消失后数据完整(meta 没被删,文件也还在) + gated.failReadPattern = null; + (reader as any).lsm.sstableCache.clear(); + (reader as any).lsm.cacheSize = 0; + expect(await reader.count('t')).toBe(30); + }); + + it('恢复报告在无损坏时为"干净",且 repair 只在干净时回收孤儿页面', async () => { + const dbName = uniqueDB('b6-repair-gate'); + const base = new MemoryBackend(); + const gated = new GatedBackend(base); + const engine = openEngine(dbName, gated); + await engine.open(dbName, 1); + const report = engine.getRecoveryReport(); + expect(report.dataLossSuspected).toBe(false); + expect(report.walGaps).toEqual([]); + await engine.close(); + }); +}); + +// =========================================================================== +// 3. WAL:分片空洞必须显式失败(不再静默丢尾部) +// =========================================================================== + +describe('[v0.8.0][B-6] WAL 分片空洞与按水位截断', () => { + async function buildSegmentedWAL(segments: number) { + const backend = new MemoryBackend(); + await backend.open('wal-gap'); + const store = new SegmentedWALStore(backend, 96); // 极小分片 → 每条记录独占一片 + const wal = new WAL(store, true, 'full'); + for (let i = 0; i < segments; i++) { + await wal.append({ + type: WALRecordType.INSERT, + txnId: 0, + tableName: 't', + key: `k${i}`, + data: { v: i, pad: 'p'.repeat(48) }, + }); + } + // 断言前提:确实一片一条(否则下面的"缺片/边界"断言就失去意义) + const segs = (await backend.listKeys()).filter((k) => /^__wal_\d+\.bin$/.test(k)).sort(); + expect(segs).toHaveLength(segments); + return { backend, store, wal }; + } + + it('活跃区间内缺失分片 → 默认抛 ARIA_WAL_GAP(不再静默丢弃尾部)', async () => { + const { backend, wal } = await buildSegmentedWAL(3); + await backend.delete('__wal_000001.bin'); // 中间挖掉一片(一片一条记录) + + const seen: unknown[] = []; + await expect(wal.recover((r) => seen.push(r))).rejects.toMatchObject({ code: 'ARIA_WAL_GAP' }); + expect(seen).toEqual([]); // 抛错时不产生"部分恢复"的副作用 + }); + + it('前缀缺失同样算空洞;显式 allowGaps 时如实上报', async () => { + const { backend, wal } = await buildSegmentedWAL(3); + await backend.delete('__wal_000000.bin'); // 缺失的是**前缀**(一片一条记录) + + await expect(wal.recover(() => { /* noop */ })).rejects.toMatchObject({ code: 'ARIA_WAL_GAP' }); + + const seen: number[] = []; + const applied = await wal.recover((r) => seen.push(r.lsn), { allowGaps: true }); + expect(applied).toBe(seen.length); + expect(wal.getLastRecoveryInfo()!.gaps).toEqual([0]); + }); + + it('按水位截断:边界保守(不误删可能含新记录的分片),水位越过全部记录后整段可删', async () => { + const { backend, store, wal } = await buildSegmentedWAL(3); + const latest = wal.getLsn(); // = 3 + + // 水位 = 1:分片 0 自身可能还含 > 1 的记录(分片内 LSN 无法从未条推出) + // → 必须整体保留(保守但绝不丢记录) + expect(await store.planKeepFrom(1, latest)).toBe(0); + // 水位 = 2:分片 0 已被完整覆盖(分片 1 的首条 = 2),分片 1 仍保守保留 + expect(await store.planKeepFrom(2, latest)).toBe(1); + // 水位 = 当前高水位:全部分片整体落盘 → 全部可删 + expect(await store.planKeepFrom(latest, latest)).toBe(3); + + await store.truncateBefore(latest, latest); + const keys = await backend.listKeys(); + expect(keys.filter((k) => k.startsWith('__wal_'))).toEqual([]); + + // 截断后分片号必须继续往前(不得复用),否则新记录会被 manifest 的 + // startSegment 跳过(实测:删除的行复活 / 已确认写入丢失) + await wal.append({ + type: WALRecordType.INSERT, txnId: 0, tableName: 't', key: 'after', data: { v: 9 }, + }); + const afterKeys = (await backend.listKeys()).filter((k) => /^__wal_\d+\.bin$/.test(k)); + expect(afterKeys).toEqual(['__wal_000003.bin']); + }); + + it('恢复按 lsn 跳过已落盘记录(旧记录不会把已删除的行复活)', async () => { + const { wal } = await buildSegmentedWAL(3); + const seen: number[] = []; + const applied = await wal.recover((r) => seen.push(r.lsn), { fromLsn: 2 }); + expect(seen).toEqual([3]); + expect(applied).toBe(1); + expect(wal.getLastRecoveryInfo()!.skipped).toBe(2); + // LSN 高水位不受跳过影响(必须继续单调,否则下一批记录会与历史 LSN 冲突) + expect(wal.getLsn()).toBe(3); + }); +}); + +// =========================================================================== +// 4. LSM 结构根治(44/45/47/49/50/51/55) +// =========================================================================== + +describe('[v0.8.0][B-6] LSM:flush 重试与错误报告顺序', () => { + it('注入一次 save 失败 → 重试后数据真的落盘(修复前失败的表再无落盘机会)', async () => { + const dbName = uniqueDB('b6-flush-retry'); + const base = new MemoryBackend(); + const failOnce = new FailOnceBackend(base); + const engine = openEngine(dbName, failOnce as unknown as IStorageBackend, { memtableSizeThreshold: 2048 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(30)); + + const lsm: any = (engine as any).lsm; + // flush 之后再 flush 一次以确保所有冻结表都有落盘机会 + await lsm.flush(); + await lsm.flush(); + expect(lsm.hasPendingFlushData()).toBe(false); + expect((await lsm.sstableStore.listMeta()).length).toBeGreaterThan(0); + // 瞬时故障必须可见(而不是静默吞掉) + expect(failOnce.remainingFailures).toBe(0); // 注入的失败确实发生过 + expect(lsm.getBackgroundWarnings().length).toBeGreaterThan(0); + // 不 close(MemoryBackend.close 会清空介质);直接在同一介质上重开验证 + + const engine2 = new AriaEngine({ storageBackend: 'memory', testBackend: base, checkpointInterval: 100_000_000 } as never); + await engine2.open(dbName, 1); + expect(await engine2.count('t')).toBe(30); + }); + + it('持续失败 → flush 明确报错且数据不丢;故障恢复后 flush 成功', async () => { + let failing = true; + let saved = 0; + const files = new Map(); + const metas: { id: number; level: number; minKey: string; maxKey: string }[] = []; + let seq = 0; + const store = { + async save(id: number, data: Uint8Array) { + if (failing) throw new Error('injected persistent save failure'); + saved++; + files.set(id, data); + return { storedSize: data.byteLength }; + }, + async load(id: number) { return files.get(id) ?? null; }, + async delete(id: number) { files.delete(id); }, + async allocateId() { return ++seq; }, + async listMeta() { return metas as never; }, + async saveMeta(m: { id: number; level: number; minKey: string; maxKey: string }) { metas.push(m); }, + async deleteMeta() { /* noop */ }, + }; + const lsm = new LSM({ memtableSizeThreshold: 128, sstableStore: store as never }); + for (let i = 0; i < 12; i++) lsm.put(`k${i}`, { v: i }); + lsm.freezeMemtable(); + await new Promise((r) => setTimeout(r, 20)); + + await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' }); + // 数据没有消失:冻结表仍在、且仍可读 + expect(lsm.getStats().frozenTables).toBeGreaterThan(0); + expect(await lsm.get('k0')).toEqual({ v: 0 }); + + // 故障消失 → flush 成功、数据落盘 + failing = false; + await lsm.flush(); + expect(lsm.getStats().frozenTables).toBe(0); + expect(saved).toBeGreaterThan(0); + expect(metas.length).toBeGreaterThan(0); + }); + + it('后台错误不得让本次 flush 白做:错误在"入链完成"之后才报告', async () => { + // 场景:后台自动 flush 失败过一次(错误已记录),随后介质恢复正常。 + // 旧实现:下一次 flush() **先检查错误** → 直接抛错,本次 flush 什么都没做 + //(待落盘数据继续只在内存里,而调用方以为它已经被处理过)。 + // 新实现:先把 memtable 入链、重试、真正落盘,然后才报告/记录那次失败。 + let failuresLeft = 1; // 第一次 save 失败,之后正常 + let saved = 0; + const files = new Map(); + const metas: { id: number; level: number }[] = []; + let seq = 0; + const store = { + async save(id: number, data: Uint8Array) { + if (failuresLeft > 0) { failuresLeft--; throw new Error('injected background save failure'); } + saved++; + files.set(id, data); + return { storedSize: data.byteLength }; + }, + async load(id: number) { return files.get(id) ?? null; }, + async delete(id: number) { files.delete(id); }, + async allocateId() { return ++seq; }, + async listMeta() { return metas as never; }, + async saveMeta(m: { id: number; level: number }) { metas.push(m); }, + async deleteMeta() { /* noop */ }, + }; + const lsm = new LSM({ memtableSizeThreshold: 64, sstableStore: store as never }); + // 触发后台自动 flush(freezeMemtable → 入链 → 第一次 save 失败 → lastBackgroundError) + for (let i = 0; i < 20; i++) lsm.put(`k${i}`, { v: i }); + lsm.freezeMemtable(); + await new Promise((r) => setTimeout(r, 30)); + expect(failuresLeft).toBe(0); // 注入的失败确实发生过 + + // 关键:这一次 flush 必须**真的执行**(早退的话什么都写不出去) + await expect(lsm.flush()).resolves.toBeUndefined(); + expect(saved).toBeGreaterThan(0); + expect(lsm.getStats().frozenTables).toBe(0); + expect(metas.length).toBeGreaterThan(0); + // 失败被重试修复 → 必须留下可见记录(不是静默吞掉) + expect(lsm.getBackgroundWarnings().length).toBeGreaterThan(0); + // 数据完整 + expect(await lsm.get('k0')).toEqual({ v: 0 }); + }); +}); + +describe('[v0.8.0][B-6/47] 流式扫描提前终止不多算', () => { + it('消费者只取 5 条时,底层来源恰好被拉取 5 条(修复前是 6 条)', () => { + const produced: number[] = []; + let pulls = 0; + const source: EntrySource = { + next() { + pulls++; + if (produced.length >= 20) return null; + const i = produced.length; + produced.push(i); + return [`k${String(i).padStart(3, '0')}`, { v: i }]; + }, + reset() { /* noop */ }, + }; + const merge = new MergeIterator(); + merge.addSource(source); + const out: string[] = []; + for (let i = 0; i < 5; i++) { + const entry = merge.next(); + if (entry) out.push(entry[0]); + } + expect(out).toHaveLength(5); + // 变异验证:把 MergeIterator.next() 里的延迟补充改回"立即 seedFromSource", + // pulls 变成 6 —— 这正是审计记录的"limit=5 却多算 1 条"。 + expect(pulls).toBe(5); + }); + + it('引擎层 limit=5 的流式扫描只产出 5 行(且提前终止)', async () => { + const dbName = uniqueDB('b6-stream-limit'); + const backend = new MemoryBackend(); + const engine = openEngine(dbName, backend); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(50)); + await (engine as any).lsm.flush(); + + const seen: Record[] = []; + const count = await engine.findStream('t', { table: 't', limit: 5 }, (row) => { seen.push(row); }); + expect(count).toBe(5); + expect(seen).toHaveLength(5); + await engine.close(); + }); +}); + +describe('[v0.8.0][B-6/49] 按层 compaction 状态(不再静默丢弃跨层触发)', () => { + it('两层同时被触发时都进入"正在 compaction"集合(单 boolean 会丢掉第二个)', async () => { + const files = new Map(); + let seq = 0; + const store = { + async save(id: number, data: Uint8Array) { files.set(id, data); return { storedSize: data.byteLength }; }, + async load(id: number) { return files.get(id) ?? null; }, + async delete(id: number) { files.delete(id); }, + async allocateId() { return ++seq; }, + async listMeta() { return [] as never; }, + async saveMeta() { /* noop */ }, + async deleteMeta() { /* noop */ }, + }; + const lsm = new LSM({ memtableSizeThreshold: 256, sstableStore: store as never }); + for (let i = 0; i < 20; i++) lsm.put(`k${String(i).padStart(3, '0')}`, { v: i }); + + // 同步连续触发两层:Set 语义下两个层号都在集合里; + // 变异验证:把 `compacting` 换回单 boolean,第二个层号会被静默丢弃。 + (lsm as unknown as { scheduleCompact(l: number): void }).scheduleCompact(0); + (lsm as unknown as { scheduleCompact(l: number): void }).scheduleCompact(1); + expect(lsm.getStats().compactingLevels).toEqual([0, 1]); + + await lsm.flush(); + expect(lsm.getStats().compactingLevels).toEqual([]); + }); +}); + +describe('[v0.8.0][B-6/50] compaction 期间该层对读者始终可见', () => { + it('合并尚未提交时,全表扫描仍能看到该层全部数据(修复前整层被 splice 掉)', async () => { + const gated = new GatedBackend(new MemoryBackend()); + const backendMap = new Map(); + const store = { + async save(id: number, data: Uint8Array) { + backendMap.set(id, data); + // 第二次 save(compaction 产物)会命中门控 → 长 await 窗口 + await gated.write(`sst_${id}`, data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer); + return { storedSize: data.byteLength }; + }, + async load(id: number) { return backendMap.get(id) ?? null; }, + async delete(id: number) { backendMap.delete(id); }, + async allocateId() { return ++allocSeq; }, + async listMeta() { return [] as never; }, + async saveMeta() { /* noop */ }, + async deleteMeta() { /* noop */ }, + }; + let allocSeq = 0; + const lsm: any = new LSM({ memtableSizeThreshold: 64, sstableStore: store as never }); + for (let i = 0; i < 40; i++) lsm.put(`k${i}`, { v: i }); + // 后台 flush 产生产物(可能已经触发 compaction,等它静默) + await lsm.flush(); + const before = await lsm.rangeScan('', '\uffff'); + expect(before.length).toBeGreaterThan(0); + + // 门控住 compaction 的产物写入 → 此时它已经取好源快照但还没提交 + let gateHit: (() => void) | null = null; + const hit = new Promise((resolve) => { gateHit = resolve; }); + gated.onGated = () => gateHit?.(); + gated.armGate('sst_'); + const compaction = lsm.compactLevel(0); + await hit; + // 核心不变量:compaction 的 await 窗口内,level 0 仍然完整可见 + const during = await lsm.rangeScan('', '\uffff'); + expect(during.map((e: [string, unknown]) => e[0])).toEqual(before.map((e: [string, unknown]) => e[0])); + gated.release(); + await compaction; + await lsm.flush(); + const after = await lsm.rangeScan('', '\uffff'); + expect(after.length).toBe(before.length); + }); + + it('并发 flush 与 compaction 交错时,扫描不会漏掉刚发布的行(结构版本重试)', async () => { + // 确定性构造那个危险窗口:读路径"取 meta 快照 → await 加载文件",而一次后台 + // flush 正好在这个 await 期间**发布新 SSTable 并从 frozenMemtables 摘掉它**。 + // 修复前这次扫描既看不到新 SSTable(不在快照里)也读不到前台冻结表 → 少行。 + const files = new Map(); + const metas: { id: number; level: number; minKey: string; maxKey: string }[] = []; + let seq = 0; + let pendingFlush: (() => Promise) | null = null; + let triggered = false; + let loads = 0; + const store = { + async save(id: number, data: Uint8Array) { + files.set(id, data); + return { storedSize: data.byteLength }; + }, + async load(id: number) { + loads++; + // 关键:在"读回源"的 await 里让后台 flush 跑完(发布新 SSTable + 摘冻结表) + if (pendingFlush && !triggered) { + triggered = true; + const flush = pendingFlush; + pendingFlush = null; + await flush(); + } + return files.get(id) ?? null; + }, + async delete(id: number) { files.delete(id); }, + async allocateId() { return ++seq; }, + async listMeta() { return metas as never; }, + async saveMeta(m: { id: number; level: number; minKey: string; maxKey: string }) { metas.push(m); }, + async deleteMeta() { /* noop */ }, + }; + const lsm: any = new LSM({ memtableSizeThreshold: 4096, sstableStore: store as never }); + // 先落一个 SSTable(保证扫描必须回源 → 进入上面的钩子) + for (let i = 0; i < 5; i++) lsm.put(`a${i}`, { v: i }); + await lsm.flush(); + expect(metas.length).toBeGreaterThan(0); + (lsm as any).sstableCache.clear(); + (lsm as any).cacheSize = 0; + (lsm as any).oversizedSSTables.clear(); + + // 后续写入的新行:扫描开始后它们才会被"后台 flush"发布出去 + for (let i = 0; i < 5; i++) lsm.put(`b${i}`, { v: 100 + i }); + pendingFlush = () => lsm.flushMemtablesOnly(); + + const during = await lsm.rangeScan('', '\uffff'); + expect(triggered).toBe(true); // 钩子确实在扫描中途触发了 flush + // 关键断言:这次扫描必须看到**全部 10 行**(既不能漏刚发布的那批, + // 也不能因为重试而重复) + expect(during.map((e: [string, unknown]) => e[0]).sort()).toEqual( + ['a0', 'a1', 'a2', 'a3', 'a4', 'b0', 'b1', 'b2', 'b3', 'b4'], + ); + + // 点查(get)也有同一份结构版本校验:这里用"回源期间前台结构发生变化" + // (写入 + 冻结)来触发它,并断言 + // ① 结果仍然正确(返回快照里的值,不因结构变化而丢数据/报错); + // ② 确实发生了一次重试(同一个 SSTable 被读了两次)。 + // 变异验证:去掉 get 里的版本校验后 loads 只增加 1 → 断言失败。 + (lsm as any).sstableCache.clear(); + (lsm as any).cacheSize = 0; + (lsm as any).oversizedSSTables.clear(); + triggered = false; + const retriesBefore = lsm.getStats().readStructureRetries; + const loadsBefore = loads; + void loadsBefore; + pendingFlush = async () => { + lsm.put('zzz', { v: 1 }); + lsm.freezeMemtable(); // 前台结构变化(活跃 memtable → frozen 列表) + }; + expect(await lsm.get('a0')).toEqual({ v: 0 }); + expect(triggered).toBe(true); + // 结构版本校验确实生效:读取期间发生前台变化 → 重试计数 +1 + //(变异验证:去掉 get 里的版本校验后计数不变 → 断言失败) + expect(lsm.getStats().readStructureRetries).toBeGreaterThan(retriesBefore); + expect(await lsm.get('zzz')).toEqual({ v: 1 }); + }); +}); + +describe('[v0.8.0][B-6/51] 底部层合并回收墓碑', () => { + it('删除密集场景:底部层合并后墓碑不再无限累积(重开后数据正确)', async () => { + const dbName = uniqueDB('b6-tombstone-gc'); + const base = new MemoryBackend(); + const engine = openEngine(dbName, base, { memtableSizeThreshold: 1024 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(120)); + await (engine as any).lsm.flush(); + await engine.delete('t', { table: 't' }); + await (engine as any).lsm.flush(); + + const lsm: any = (engine as any).lsm; + const countTombstones = async (): Promise => { + const metas = await lsm.sstableStore.listMeta(); + let tombstones = 0; + for (const meta of metas) { + const data = await lsm.sstableStore.load(meta.id); + if (!data) continue; + new SSTableReader(data, meta).scanAll((_k, v) => { + if ((v as Record).__tombstone) tombstones++; + }); + } + return tombstones; + }; + + // 把数据/墓碑一路压到**底部层**:每层用 minFiles=1 逐级下沉 + const compact = (level: number, minFiles?: number) => + (lsm as unknown as { compactLevelAsync(l: number, m?: number): Promise }) + .compactLevelAsync(level, minFiles); + for (let level = 0; level < MAX_LSM_LEVELS - 1; level++) { + // 反复压,直到该层没有文件为止(每轮产物落到下一层) + for (let round = 0; round < 8 && lsm.levels[level].length > 0; round++) { + await compact(level, 1); + } + } + const bottomBefore = lsm.levels[MAX_LSM_LEVELS - 1].length; + expect(bottomBefore).toBeGreaterThan(0); // 数据确实到了底部层 + const beforeGc = await countTombstones(); + expect(beforeGc).toBeGreaterThan(0); // 底部层合并前墓碑确实存在 + + // 触发底部层原地合并(drop tombstones) + await compact(MAX_LSM_LEVELS - 1, 1); + const afterGc = await countTombstones(); + // 变异验证:去掉 isBottomLevel 的墓碑过滤,这里会等于 beforeGc + expect(afterGc).toBe(0); + + // 语义不变:删除仍然生效(没有因为丢墓碑而复活) + expect(await engine.count('t')).toBe(0); + + // 不 close(MemoryBackend.close 会清空介质);同一介质上重开验证持久化 + const engine2 = new AriaEngine({ storageBackend: 'memory', testBackend: base, checkpointInterval: 100_000_000 } as never); + await engine2.open(dbName, 1); + expect(await engine2.count('t')).toBe(0); + }); +}); + +describe('[v0.8.0][B-6/55] checkpoint 不再等完整 compaction', () => { + it('compaction 卡住时,写路径的周期 checkpoint 仍能完成并落盘', async () => { + const dbName = uniqueDB('b6-tick-decouple'); + const base = new MemoryBackend(); + const gated = new GatedBackend(base); + const engine = openEngine(dbName, gated, { + memtableSizeThreshold: 512, + checkpointInterval: 1, // 每次写入都尝试 checkpoint + }); + await engine.open(dbName, 1); + // 带 pad 列:每行都超过 memtable 阈值 → 每次 insert 产生一个 level-0 SSTable + await engine.createTable(createSchema('t', { + id: { type: 'string', primaryKey: true }, + v: { type: 'number' }, + pad: { type: 'string' }, + })); + + const lsm: any = (engine as any).lsm; + const big = 'x'.repeat(600); + // 先造 3 个 level-0 SSTable(3 < 4 → 不会自动触发 compaction) + for (let i = 0; i < 3; i++) { + await engine.insert('t', [{ id: `k${i}`, v: i, pad: big }]); + await lsm.flushMemtablesOnly(); + } + expect(lsm.levels[0].length).toBe(3); + + // 门控:放行第 4 次 flush 自己的 save,阻塞随后自动触发的 compaction 的 save + let gateHit: (() => void) | null = null; + const hit = new Promise((resolve) => { gateHit = resolve; }); + gated.onGated = () => gateHit?.(); + gated.armGate('sst_', 1); + await engine.insert('t', [{ id: 'k3', v: 3, pad: big }]); + await lsm.flushMemtablesOnly(); + const gatedOutcome = await Promise.race([ + hit.then(() => 'gated'), + new Promise((r) => setTimeout(() => r('no-compaction'), 5000)), + ]); + expect(gatedOutcome).toBe('gated'); // compaction 确实开始了且被卡住 + + // compaction 现在卡在门控上(维护链不前进)。写路径必须继续工作: + // 修复前 checkpoint → lsm.flush() → drainMaintenance() 会一起卡住 + //(v0.6.1 记录的"8~11s 悬崖"的成因之一)。 + const outcome = await Promise.race([ + engine.insert('t', [{ id: 'after-gate', v: 999, pad: big }]).then(() => 'ok', () => 'error'), + new Promise((r) => setTimeout(() => r('BLOCKED'), 5000)), + ]); + expect(outcome).toBe('ok'); + + gated.release(); + await lsm.flush(); + expect(await engine.count('t')).toBe(5); + }); +}); + +describe('[v0.8.0][B-6] 冻结表意图:已确认写入不得被水位越过', () => { + it('manifest 声称有未落盘冻结表、但 WAL 里什么都没有 → 打开时报 ARIA_WRITE_LOST', async () => { + const dbName = uniqueDB('b6-intent-lost'); + const backend = new MemoryBackend(); + await backend.open(dbName); + // 手工构造:一份声称"有冻结表"的 manifest + 没有任何 WAL 记录 + const store = new ManifestStore({ backend, instanceId: 'crafted' }); + await store.load(); + const crafted: AriaManifest = { + ...store.current, + frozen: [{ + ns: 'main', + id: 1, + entryCount: 7, + minKey: 't:k1', + maxKey: 't:k7', + lsnAtFreeze: 3, + }], + wal: { startSegment: 0, startLsn: 3, nextLsn: 5 }, + }; + const bytes = encodeManifest({ ...crafted, generation: 1 }); + await backend.write(manifestKey(1), bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer); + + const engine = new AriaEngine({ storageBackend: 'memory', testBackend: backend, checkpointInterval: 100_000_000 } as never); + await expect(engine.open(dbName, 1)).rejects.toMatchObject({ code: 'ARIA_WRITE_LOST' }); + }); + + it('冻结表存在时,水位不会推进越过它(manifest 如实记录意图)', async () => { + const dbName = uniqueDB('b6-intent-floor'); + const backend = new MemoryBackend(); + const engine = openEngine(dbName, backend, { memtableSizeThreshold: 128 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + await engine.insert('t', rows(20)); + // 强行再冻结一份(不 flush),让"待落盘意图"存在 + (engine as any).lsm.freezeMemtable(); + + const manifest = (await readManifestState(backend))!; + // 有意让意图进入 manifest(提交一次),断言水位 <= 最早意图的 lsnAtFreeze + await (engine as any).commitManifest(); + const after = (await readManifestState(backend))!; + void manifest; + expect(after.frozen.length).toBeGreaterThan(0); + const minIntentLsn = Math.min(...after.frozen.map((f) => f.lsnAtFreeze)); + expect(after.wal.startLsn).toBeLessThanOrEqual(minIntentLsn); + await engine.close(); + }); +}); + +describe('[v0.8.0][B-6] 页面映射:已退休的 SSTable 仍可被在途读者读取', () => { + it('compaction 摘除 meta 后,旧页面映射保留到物理删除', async () => { + const dbName = uniqueDB('b6-retire'); + const backend = new MemoryBackend(); + const engine = openEngine(dbName, backend, { memtableSizeThreshold: 1024 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + for (let i = 0; i < 60; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]); + const lsm: any = (engine as any).lsm; + await lsm.flush(); + const before = await lsm.sstableStore.listMeta(); + await lsm.compactLevel(0); + + // 被取代的 meta 已从 manifest 摘除 + const after = await readManifestState(backend); + const afterIds = new Set(after!.namespaces.main.sstables.map((m) => m.id)); + expect(before.some((m: { id: number }) => !afterIds.has(m.id))).toBe(true); + // 但数据仍完整可读(退休文件在"没有更早读者"之前不物理删除) + expect(await engine.count('t')).toBe(60); + await engine.close(); + }); +}); + +describe('[v0.8.0][B-6] SSTable 解析:三个读取路径越界策略一致', () => { + it('三个读取路径对同一个损坏文件给出一致结论(修复前三份实现各自为政)', () => { + const { SSTableBuilder } = require('../src/engine/aria/index/sstable_builder'); + const builder = new SSTableBuilder(64); // 小块 → 多个块 + const total = 30; + for (let i = 0; i < total; i++) builder.add(`k${String(i).padStart(3, '0')}`, { v: i }); + const { sstableData } = builder.build(); + const meta = { + id: 1, level: 0, minKey: 'k000', maxKey: 'k029', + blockCount: 1, totalSize: sstableData.byteLength, bloomData: null, + } as never; + + // 健康读取器:用于确定"第一个块覆盖到哪个 key" + const healthy = new SSTableReader(sstableData, meta); + const indexEntries = (healthy as unknown as { indexEntries: { key: string; blockOffset: number; blockSize: number }[] }) + .indexEntries; + expect(indexEntries.length).toBeGreaterThan(1); // 前提:确实有多个块 + const firstBlockLastKey = indexEntries[0].key; + const healthyKeys: string[] = []; + healthy.scanAll((k) => { healthyKeys.push(k); }); + const expectedAfterFirstBlock = healthyKeys.filter((k) => k > firstBlockLastKey); + expect(expectedAfterFirstBlock.length).toBeGreaterThan(0); + + // 破坏第一个块里**第一个条目的 keyLen 字段**(块布局:[u32 条目数][u32 keyLen]...): + // 长度字段越界 → 按统一策略"本块剩余条目整体放弃、继续后续块"。 + // (注意不能只改块尾部:尾部是最后一个条目的**值**字节,坏掉只会让那条记录 + // 解析失败,不会触发越界分支 —— 那样就测不到三条路径的策略是否一致。) + const damaged = new Uint8Array(sstableData); + const block = indexEntries[0]; + damaged.fill(0xff, block.blockOffset + 4, block.blockOffset + 8); + const damagedReader = new SSTableReader(damaged, meta); + + // 三条读取路径都不许抛异常(统一越界策略 = 跳过越界部分) + const viaScanAll: string[] = []; + expect(() => damagedReader.scanAll((k) => { viaScanAll.push(k); })).not.toThrow(); + const viaScanLazy: string[] = []; + expect(() => { + for (const [k] of damagedReader.scanLazy('', '\uffff')) viaScanLazy.push(k); + }).not.toThrow(); + // get:损坏块里的 key 读不到(跳过),但不抛错 + const inDamagedBlock = damagedReader.get(firstBlockLastKey); + expect(inDamagedBlock).toBeNull(); + + // 一致结论 1:两个扫描路径给出**同一集合**(同一份解析实现) + expect([...viaScanAll].sort()).toEqual([...viaScanLazy].sort()); + // 一致结论 2:损坏块**之后**的块仍然完整可读(越界只影响该块剩余条目, + // 不是"整个文件作废" —— 变异验证:把 iterEntries 里的 break 换成 return, + // 这里的后续块条目会全部消失) + for (const key of expectedAfterFirstBlock) { + expect(viaScanAll).toContain(key); + } + // 一致结论 3:点查在后继块里仍然命中 + const lastKey = `k${String(total - 1).padStart(3, '0')}`; + expect(damagedReader.get(lastKey)).toEqual({ v: total - 1 }); + }); +}); + +// =========================================================================== +// 5. 维护路径:vacuum / close 的收尾语义 / 恢复报告聚合 +// =========================================================================== + +describe('[v0.8.0][B-6] 维护路径的如实语义', () => { + it('vacuum 返回**真实**压缩层数(修复前硬编码 6,且底部层永不压缩)', async () => { + const dbName = uniqueDB('b6-vacuum'); + const base = new MemoryBackend(); + const engine = openEngine(dbName, base, { memtableSizeThreshold: 512 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + + // 空库:没有任何层需要压缩 → 必须是 0(硬编码 6 会立刻失败) + const empty = await engine.vacuum(); + expect(empty.compactedLevels).toBe(0); + + // 造多个 SSTable → 至少一层文件数 ≥ 2 → 真实压缩 + for (let i = 0; i < 30; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]); + await (engine as any).lsm.flush(); + const result = await engine.vacuum(); + expect(result.compactedLevels).toBeGreaterThan(0); + expect(await engine.count('t')).toBe(30); + }); + + it('vacuum 在删除密集后回收墓碑(底部层原地合并)', async () => { + const dbName = uniqueDB('b6-vacuum-gc'); + const base = new MemoryBackend(); + const engine = openEngine(dbName, base, { memtableSizeThreshold: 512 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + for (let i = 0; i < 40; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]); + await (engine as any).lsm.flush(); + await engine.delete('t', { table: 't' }); + await (engine as any).lsm.flush(); + + const lsm: any = (engine as any).lsm; + // 把数据/墓碑压到底部层,然后 vacuum 触发原地合并(回收墓碑) + for (let level = 0; level < MAX_LSM_LEVELS - 1; level++) { + for (let round = 0; round < 8 && lsm.levels[level].length > 0; round++) { + await lsm.compactLevelAsync(level, 1); + } + } + await engine.vacuum(); + let tombstones = 0; + for (const meta of await lsm.sstableStore.listMeta()) { + const data = await lsm.sstableStore.load(meta.id); + if (!data) continue; + new SSTableReader(data, meta).scanAll((_k, v) => { + if ((v as Record).__tombstone) tombstones++; + }); + } + expect(tombstones).toBe(0); + expect(await engine.count('t')).toBe(0); + }); + + it('close() 在落盘失败时仍必须释放后端/锁并复位状态(修复前直接卡在 flush 上)', async () => { + const dbName = uniqueDB('b6-close-finally'); + const base = new MemoryBackend(); + const gated = new GatedBackend(base); + const engine = openEngine(dbName, gated, { memtableSizeThreshold: 512, pageStorage: true }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + for (let i = 0; i < 10; i++) await engine.insert('t', [{ id: `k${i}`, v: i, }]); + // 让落盘必然失败:页面写全部抛错 + gated.failWritePattern = 'pg_'; + await expect(engine.close()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' }); + // 关键:即便 flush 失败,后端/锁/运行期状态都必须已经收尾 + expect(engine.isOpen()).toBe(false); + expect((engine as any).dbLock).toBeNull(); + expect((engine as any).schemas.size).toBe(0); + }); + + it('恢复报告聚合 LSM 侧被丢弃的 SSTable(含命名空间与原因)', async () => { + const dbName = uniqueDB('b6-report'); + const base = new MemoryBackend(); + const writer = openEngine(dbName, base, { memtableSizeThreshold: 64 * 1024 * 1024, pageStorage: true }); + await writer.open(dbName, 1); + await writer.createTable(SCHEMA()); + await writer.insert('t', rows(20)); + await (writer as any).lsm.flush(); + + // 删掉一个页面文件 → 该 SSTable 不可用(元数据仍在 manifest 里) + const manifest = (await readManifestState(base))!; + const victimPages = manifest.namespaces.main.sstables[0].pageIds ?? []; + expect(victimPages.length).toBeGreaterThan(0); + await base.deleteMany(victimPages.map((pid) => `pg_${pid}`)); + + const reader = openEngine(dbName, base, { pageStorage: true }); + await reader.open(dbName, 1); + const report = reader.getRecoveryReport(); + expect(report.droppedSSTables.length).toBeGreaterThan(0); + expect(report.droppedSSTables[0].namespace).toBe('main'); + expect(report.droppedSSTables[0].reason.length).toBeGreaterThan(0); + // 数据丢了 → 必须被标记出来(不是静默少几行) + expect(report.dataLossSuspected).toBe(true); + }); + + it('repair() 强制回收退休文件(退休登记清零)', async () => { + const dbName = uniqueDB('b6-retire-repair'); + const base = new MemoryBackend(); + const engine = openEngine(dbName, base, { memtableSizeThreshold: 512 }); + await engine.open(dbName, 1); + await engine.createTable(SCHEMA()); + for (let i = 0; i < 40; i++) await engine.insert('t', [{ id: `k${i}`, v: i }]); + const lsm: any = (engine as any).lsm; + await lsm.flush(); + await lsm.compactLevel(0); + await engine.repair(); + expect(lsm.getRetiredCount()).toBe(0); + expect(await engine.count('t')).toBe(40); + }); +}); + +// =========================================================================== +// 6. manifest 严格校验:每一种元数据损坏都必须被**明确拒绝** +// =========================================================================== + +describe('[v0.8.0][B-6] manifest 严格校验(任一字段损坏都必须判世代无效)', () => { + /** 用任意(可能非法的)载荷 JSON 拼出一个"头部/CRC 都正确"的 manifest 字节流 */ + function craft(payload: unknown): Uint8Array { + const json = new TextEncoder().encode(JSON.stringify(payload)); + const buf = new ArrayBuffer(MANIFEST_HEADER_SIZE + json.byteLength); + const view = new DataView(buf); + view.setUint32(0, 0x4d534d46, false); // magic(与生产一致) + view.setUint16(4, 1, false); // formatVersion + view.setUint16(6, MANIFEST_HEADER_SIZE, false); + view.setUint32(8, 1, false); // generation + view.setUint32(12, json.byteLength, false); + view.setUint32(16, crc32(json), false); + view.setUint32(20, crc32(new Uint8Array(buf, 0, 20)), false); + new Uint8Array(buf, MANIFEST_HEADER_SIZE).set(json); + return new Uint8Array(buf); + } + + /** 一份合法的载荷(各用例只改一个字段) */ + function validPayload(): Record { + return { + formatVersion: 1, + generation: 1, + pageIdWatermark: 7, + namespaces: { + main: { + nextSstableId: 3, + sstables: [{ + id: 2, level: 0, minKey: 't:a', maxKey: 't:z', + blockCount: 1, totalSize: 128, bloomData: null, pageIds: [1, 2], + }], + }, + }, + schemas: { t: { id: { type: 'string', primaryKey: true } } }, + wal: { startSegment: 0, startLsn: 4, nextLsn: 9 }, + frozen: [{ + ns: 'main', id: 1, entryCount: 3, minKey: 't:a', maxKey: 't:c', lsnAtFreeze: 4, + }], + owner: { instanceId: 'x', epoch: 1, openedAt: 1000 }, + committedAt: 1000, + }; + } + + const cases: [string, (p: Record) => void, string][] = [ + ['formatVersion 不支持', (p) => { p.formatVersion = 2; }, 'formatVersion'], + ['generation 与头部不一致', (p) => { p.generation = 42; }, 'generation mismatch'], + ['pageIdWatermark 非法', (p) => { p.pageIdWatermark = -1; }, 'pageIdWatermark'], + ['namespaces 不是对象', (p) => { p.namespaces = 5; }, 'namespaces is not an object'], + ['命名空间不是对象', (p) => { p.namespaces = { main: 7 }; }, 'is not an object'], + ['nextSstableId 非法', (p) => { (p.namespaces as any).main.nextSstableId = -2; }, 'nextSstableId'], + ['sstables 不是数组', (p) => { (p.namespaces as any).main.sstables = {}; }, 'sstables is not an array'], + ['sstable 条目不是对象', (p) => { (p.namespaces as any).main.sstables = [3]; }, 'non-object sstable'], + ['sstable.id 非法', (p) => { (p.namespaces as any).main.sstables[0].id = 'x'; }, 'sstables[].id'], + ['sstable.totalSize 非法', (p) => { (p.namespaces as any).main.sstables[0].totalSize = -1; }, 'totalSize'], + ['sstable.minKey 非字符串', (p) => { (p.namespaces as any).main.sstables[0].minKey = 5; }, 'minKey'], + ['sstable.pageIds 不是数组', (p) => { (p.namespaces as any).main.sstables[0].pageIds = 5; }, 'pageIds is not an array'], + ['sstable.pageIds 元素非法', (p) => { (p.namespaces as any).main.sstables[0].pageIds = [-1]; }, 'pageIds[0]'], + ['schemas 不是对象', (p) => { p.schemas = 5; }, 'schemas is not an object'], + ['schemas.
不是对象', (p) => { p.schemas = { t: 5 }; }, 'is not an object'], + ['wal 不是对象', (p) => { p.wal = 5; }, 'wal is not an object'], + ['wal.startSegment 非法', (p) => { (p.wal as any).startSegment = -1; }, 'wal.startSegment'], + ['wal.startLsn > nextLsn', (p) => { (p.wal as any).startLsn = 99; }, 'startLsn'], + ['frozen 不是数组', (p) => { p.frozen = {}; }, 'frozen is not an array'], + ['frozen 条目不是对象', (p) => { p.frozen = [1]; }, 'non-object entry'], + ['frozen[].lsnAtFreeze 非法', (p) => { (p.frozen as any)[0].lsnAtFreeze = -5; }, 'lsnAtFreeze'], + ['owner 不是对象', (p) => { p.owner = 5; }, 'owner is not an object'], + ['owner.instanceId 非字符串', (p) => { (p.owner as any).instanceId = 5; }, 'owner.instanceId'], + ['committedAt 非法', (p) => { p.committedAt = -1; }, 'committedAt'], + ]; + + it.each(cases)('%s → 该世代无效且给出原因', (_name, mutate, reason) => { + const payload = validPayload(); + mutate(payload); + const decoded = decodeManifest(craft(payload)); + expect(decoded.ok).toBe(false); + if (!decoded.ok) expect(decoded.reason).toContain(reason); + }); + + it('载荷不是对象 / 不是合法 JSON → 判无效(不抛异常)', () => { + const asArray = decodeManifest(craft([1, 2, 3])); + expect(asArray.ok).toBe(false); + const notJson = new Uint8Array(encodeManifest({ + ...createEmptyManifest({ instanceId: 'x' }), generation: 1, + })); + // 把载荷区改成非法 JSON(保留头部与 CRC 之外的部分:这里直接改 CRC 覆盖的载荷) + const bad = craft({}); // 合法 JSON 但缺字段 + expect(decodeManifest(bad).ok).toBe(false); + expect(decodeManifest(notJson).ok).toBe(true); // 对照组:正常字节流可解码 + }); + + it('字节流层面的损坏:过短 / 魔数错 / 版本不支持 / 头部 CRC 错 / 长度非法 / 载荷 CRC 错', () => { + const good = encodeManifest({ ...createEmptyManifest({ instanceId: 'x' }), generation: 1 }); + + const tooSmall = decodeManifest(good.slice(0, 10)); + expect(tooSmall.ok).toBe(false); + if (!tooSmall.ok) expect(tooSmall.reason).toContain('too small'); + + const badMagic = new Uint8Array(good); + new DataView(badMagic.buffer).setUint32(0, 0xdeadbeef, false); + // 头部 CRC 也会因此失配 + expect(decodeManifest(badMagic).ok).toBe(false); + + // 版本不支持:改版本号但重算头部 CRC,确保命中"版本"分支 + const badVersion = new Uint8Array(good); + const vView = new DataView(badVersion.buffer); + vView.setUint16(4, 9, false); + vView.setUint32(20, crc32(badVersion.subarray(0, 20)), false); + const versionResult = decodeManifest(badVersion); + expect(versionResult.ok).toBe(false); + if (!versionResult.ok) expect(versionResult.reason).toContain('unsupported format version'); + + // 头部尺寸不一致 + const badHeaderSize = new Uint8Array(good); + const hView = new DataView(badHeaderSize.buffer); + hView.setUint16(6, 99, false); + hView.setUint32(20, crc32(badHeaderSize.subarray(0, 20)), false); + const headerResult = decodeManifest(badHeaderSize); + expect(headerResult.ok).toBe(false); + if (!headerResult.ok) expect(headerResult.reason).toContain('header size'); + + // 载荷长度非法(大于实际字节数) + const badLength = new Uint8Array(good); + const lView = new DataView(badLength.buffer); + lView.setUint32(12, 10 ** 6, false); + lView.setUint32(20, crc32(badLength.subarray(0, 20)), false); + const lengthResult = decodeManifest(badLength); + expect(lengthResult.ok).toBe(false); + if (!lengthResult.ok) expect(lengthResult.reason).toContain('payload length'); + + // 载荷 CRC 失配 + const badCrc = new Uint8Array(good); + badCrc[badCrc.byteLength - 1] ^= 0xff; + const crcResult = decodeManifest(badCrc); + expect(crcResult.ok).toBe(false); + if (!crcResult.ok) expect(crcResult.reason).toContain('payload CRC mismatch'); + }); + + it('未 load 就 commit → ARIA_MANIFEST_NOT_LOADED(不写坏介质)', async () => { + const backend = new MemoryBackend(); + await backend.open('not-loaded'); + const store = new ManifestStore({ backend, instanceId: 'x' }); + await expect(store.commit()).rejects.toMatchObject({ code: 'ARIA_MANIFEST_NOT_LOADED' }); + expect((await backend.listKeys()).filter((k) => k.startsWith('__aria_manifest_'))).toEqual([]); + }); +});