feat(B-6): 存储层单一提交点(__aria_manifest)+ LSM 结构根治

按 PLAN-v0.7.5.md §B-6 的**完整规格**实施(此前只落地了"降级选项"里的五处止血):
B-6 要求的是 `__aria_manifest` 单一提交点 + LSM 单项改造。完整记录见方案附录 H。

一、单一提交点
  - 新增 `src/engine/aria/store/manifest.ts`:`__aria_manifest_<generation>`
    (magic + formatVersion + generation + 头部 CRC + 载荷 CRC;先写后验;保留两代)。
    载荷 = 页面水位 + 各命名空间 SSTable 元数据 + 表结构 + WAL 起始位置 + 待落盘冻结表意图。
  - 顺序固定:**数据落盘 → manifest 提交 → 才允许截断 WAL / 删除旧文件 / 删除旧 SSTable**。
  - 恢复只认最后一份 CRC 通过的世代;全部世代无效 → `ARIA_MANIFEST_CORRUPT`
    (修复前:裸 JSON meta 解析失败 → `[]` → 静默空库,随后 repair 还会删光活页)。
  - 旧格式(__aria_lsm_meta/__aria_schemas/__aria_meta)首次打开自动迁移,旧键保留;
    迁移遇到损坏 → `ARIA_LEGACY_META_CORRUPT`。
  - 陈旧实例保护(STALE_INSTANCE):认领时一次跨过 MANIFEST_TAKEOVER_STRIDE 个世代,
    杜绝"旧实例在途提交落在同一世代号上"(实测第二个实例 open 直接失败)。

二、LSM
  - 44 冻结表成为一等状态:失败保留 + 可重试(修复前失败即永久失去落盘机会)。
  - 45 `flush()` 先入链再报告后台错误(修复前一次后台失败会让之后每次 flush 直接抛错、
       数据永远等不到落盘);被重试修复的失败进 `getBackgroundWarnings()`(可见但不误报失败)。
  - 47 `MergeIterator` 胜出来源的补充推迟到下一次 `next()`:提前终止不再多算一条。
  - 49 `compacting` 由单 boolean 改为按层集合(跨层触发不再被静默丢弃)。
  - 50 compaction 不再"先 splice 整层再合并"(窗口内该层对读者可见);
       被取代的 SSTable 进"退休表" + 读者 epoch,等更早读者退出才物理删除。
  - 51 底部层原地合并回收墓碑(删除密集场景空间不再无界增长);"整层只剩墓碑" 有专门分支
       (修复前会读 `merged[0][0]` 抛 TypeError,compaction 永久失败)。
  - 55 flush 与 compaction 拆成两条链,checkpoint 只落 memtable;删除引擎层全部
       `prefetch*`/`drainChain` 依赖,改为"快照 + 结构版本乐观重试"
       (版本号同时覆盖 levels 与前台 memtable/frozen 的变化)。
  - 读路径自洽:介质读故障抛 `ARIA_SSTABLE_READ_FAILED`,不再折叠成"文件不存在"误删元数据。

三、WAL
  - LSN 全库单调(manifest 记高水位);按水位删除旧分片(`planKeepFrom` → 提交 → 再删除)。
  - **分片号只增不减**:修复前全量截断后重置为 0,会与 manifest 记录的 startSegment 错位,
    实测造成两个方向的损坏(删掉的行复活 / 已确认写入丢失,见随机压力套件)。
  - 分片空洞(含前缀缺失)显式报 `ARIA_WAL_GAP`,不再静默丢弃尾部。

四、其它
  - `sstable.ts` 三份解析循环合并为 `iterEntries()`,越界策略统一。
  - `vacuum()` 返回真实压缩层数(修复前硬编码 6 且底部层永不压缩)。
  - `close()` 加 try/finally(落盘失败也必须释放后端/锁并复位状态)。
  - `getRecoveryReport()`:{droppedSSTables, dataLossSuspected, walGaps, legacyImported,
    manifestFallback} —— "自愈了什么、有没有真丢数据"成为可读返回值。

五、验证
  - 新增 `tests/v080-b6-single-commit-point.test.ts`(63 项,含 manifest 严格校验表驱动 25 例)。
  - 新增 `scripts/mutation-b6.py`:22 项变异验证(把每个修复回退到修复前行为,对应用例必须失败),
    全部被拦住 —— 这批用例不是陪跑。
  - 常规套件 1935 通过 / 91 套件;覆盖率 90.34 / 82.16 / 94.06 / 93.23(阈值 90/82/94/93);
    e2e 14/14;重型套件 4 套件 27 项全绿。
This commit is contained in:
thzxx
2026-09-15 10:29:03 +08:00
parent 714e7f98a4
commit c5694b1d23
20 changed files with 4764 additions and 709 deletions
+373
View File
@@ -0,0 +1,373 @@
#!/usr/bin/env python3
"""B-6 变异验证:把每个修复回退到修复前的行为,对应用例必须失败。
用法:python3 scripts/mutation-b6.py
任何一条"回退后测试仍然通过"都会以非零退出码报出来(说明测试只是陪跑)。
"""
import io
import os
import re
import shlex
import signal
import subprocess
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LSM = 'src/engine/aria/index/lsm.ts'
MANIFEST = 'src/engine/aria/store/manifest.ts'
MERGE = 'src/engine/aria/index/merge_iterator.ts'
ENGINE = 'src/engine/aria/index.ts'
SEGSTORE = 'src/engine/aria/wal/segmented_store.ts'
CHECKPOINT = 'src/engine/aria/wal/checkpoint.ts'
SSTABLE = 'src/engine/aria/index/sstable.ts'
B6 = 'tests/v080-b6-single-commit-point.test.ts'
STRESS = 'tests/engine/aria-repair-hardening.test.ts'
MUTATIONS = [
dict(
name='47 MergeIterator 立即补充(提前终止多算 1 条)',
file=MERGE,
old=""" // 胜出来源的补充推迟到下一次 next()(消费者只取 N 条 → 底层只产出 N 条)
this.pendingRefill = first.sourceIndex;""",
new=""" // [MUTATION] 立即补充(修复前行为)
this.seedFromSource(first.sourceIndex);""",
test=B6, pattern='流式扫描提前终止不多算',
),
dict(
name='49 compacting 单 boolean(跨层触发被丢弃)',
file=LSM,
old=""" private scheduleCompact(level: number): void {
if (level > MAX_LSM_LEVELS - 1 || this.compacting.has(level)) return;
this.compacting.add(level);""",
new=""" private scheduleCompact(level: number): void {
if (level > MAX_LSM_LEVELS - 1 || this.compacting.size > 0) return; // [MUTATION] 单 boolean 语义
this.compacting.add(level);""",
test=B6, pattern='按层 compaction 状态',
),
dict(
name='50 compaction 先 splice 整层(窗口内该层不可见)',
file=LSM,
old=""" const selected = [...this.levels[level]];
const selectedIds = new Set(selected.map((m) => m.id));""",
new=""" const selected = this.levels[level].splice(0, this.levels[level].length); // [MUTATION] 旧行为
const selectedIds = new Set(selected.map((m) => m.id));""",
test=B6, pattern='compaction 期间该层对读者始终可见',
),
dict(
name='51 底部层不回收墓碑',
file=LSM,
old=""" const merged = isBottomLevel
? mergedRaw.filter(([, value]) => !(value as unknown as Record<string, unknown>).__tombstone)
: mergedRaw;""",
new=""" const merged = mergedRaw; // [MUTATION] 不回收墓碑""",
test=B6, pattern='底部层合并回收墓碑',
),
dict(
name='44 冻结表失败后不重试',
file=LSM,
old=""" private async retryPendingFlushes(): Promise<void> {
let rounds = 0;""",
new=""" private async retryPendingFlushes(): Promise<void> {
return; // [MUTATION] 无重试路径
let rounds = 0;""",
test=B6, pattern='持续失败 → flush 明确报错且数据不丢',
),
dict(
name='45 后台错误检查放回"入链之前"',
file=LSM,
old=""" async flush(): Promise<void> {
this.enqueuePendingMemtables();
const reported = this.consumeBackgroundError();
await this.drainChain();""",
new=""" async flush(): Promise<void> {
// [MUTATION] 旧行为:先检查/报告后台错误 → 本次 flush 被整个跳过
const early = this.consumeBackgroundError();
if (early !== null) {
throw new DatabaseError('background error (legacy behaviour)', 'ARIA_BACKGROUND_ERROR', early);
}
this.enqueuePendingMemtables();
const reported = this.consumeBackgroundError();
await this.drainChain();""",
test=B6, pattern='后台错误不得让本次 flush 白做',
),
dict(
name='manifest 全世代损坏 → 返回空状态(静默空库)',
file=MANIFEST,
old=""" if (generations.length > 0) {
// 文件存在但一代都读不出来:这是元数据损坏,不是空库
throw new DatabaseError(""",
new=""" if (false && generations.length > 0) { // [MUTATION] 静默当空库
throw new DatabaseError(""",
test=B6, pattern='全部世代都损坏',
),
dict(
name='陈旧实例不设防(静默覆盖新世代)',
file=MANIFEST,
old=""" if (!opts.allowTakeover && newestForeign > this.generation) {""",
new=""" if (false && !opts.allowTakeover && newestForeign > this.generation) { // [MUTATION] 不设防""",
test=B6, pattern='陈旧实例提交被拒绝',
),
dict(
name='旧格式 meta 损坏 → 返回空(静默空库)',
file=ENGINE,
old=""" } catch (error) {
throw new DatabaseError(
`Legacy SSTable metadata "${key}" is corrupt and cannot be migrated: ${(error as Error).message}`,
'ARIA_LEGACY_META_CORRUPT',
error,
);
}""",
new=""" } catch (error) {
continue; // [MUTATION] 静默跳过损坏的旧 meta
}""",
test=B6, pattern='旧格式 meta 损坏',
),
dict(
name='介质读故障被当成"文件不存在"(误删 meta',
file=LSM,
old=""" } catch (error) {
throw new DatabaseError(
`AriaEngine failed to read SSTable id=${meta.id} from storage: ${(error as Error).message}`,
'ARIA_SSTABLE_READ_FAILED',
error,
);
}""",
new=""" } catch (error) {
await this.dropInvalidSSTable(meta, 'read failure treated as missing'); // [MUTATION] 旧行为
return null;
}""",
test=B6, pattern='介质读故障 ≠ 文件缺失',
),
dict(
name='WAL 分片空洞静默丢弃尾部',
file='src/engine/aria/wal/log.ts',
old=""" if (gaps.length > 0 && !opts.allowGaps) {""",
new=""" if (false && gaps.length > 0 && !opts.allowGaps) { // [MUTATION] 静默丢弃""",
test=B6, pattern='活跃区间内缺失分片',
),
dict(
name='WAL 分片号复用(旧世代排在新记录之后)',
file=SEGSTORE,
old=""" if (leftover.length === 0) {
this.currentSegment = Math.max(maxSeq + 1, keepFrom);""",
new=""" if (leftover.length === 0) {
this.currentSegment = 0; // [MUTATION] 复用分片号""",
test=STRESS, pattern='随机操作压力',
),
dict(
name='冻结表意图不做"写入是否真的丢了"校验',
file=ENGINE,
old=""" const summary = intents.map((i) => `${i.ns}#${i.id}(${i.entryCount} 项)`).join(', ');
throw new DatabaseError(""",
new=""" const summary = intents.map((i) => `${i.ns}#${i.id}(${i.entryCount} 项)`).join(', ');
if (summary) return; // [MUTATION] 不校验
throw new DatabaseError(""",
test=B6, pattern='ARIA_WRITE_LOST',
),
dict(
name='提交中的冻结表仍写进意图(回退时误报 WRITE_LOST)',
file=LSM,
old=""" // 正在提交的表由当前这次 manifest 提交负责(见 FrozenTable.committing
if (frozen.committing) continue;""",
new=""" // [MUTATION] 提交中的表也写进意图""",
test=B6, pattern='回退上一代并标记 manifestFallback',
),
dict(
name='checkpoint 等完整 compaction(写路径被拖住)',
file=CHECKPOINT,
old=""" if (this.flushable && typeof this.flushable.flushMemtables === 'function') {""",
new=""" if (false && this.flushable && typeof this.flushable.flushMemtables === 'function') { // [MUTATION] 旧行为""",
test=B6, pattern='checkpoint 不再等完整 compaction',
),
dict(
name='collectReaders 不做结构版本重试(扫描漏掉并发发布的 SSTable)',
file=LSM,
old=""" if (version === this.structureVersion) return readers;
this.readStructureRetries++;
}""",
new=""" return readers; // [MUTATION] 不校验结构版本
}""",
test=B6, pattern='并发 flush 与 compaction 交错时',
),
dict(
name='点查不做结构版本重试(get 的校验失效)',
file=LSM,
old=""" if (version !== this.structureVersion) {
this.readStructureRetries++;
continue; // 期间有结构变化(新产物发布 / 前台冻结)→ 重来
}""",
new=""" if (false) {
this.readStructureRetries++;
continue;
}""",
test=B6, pattern='并发 flush 与 compaction 交错时',
),
dict(
name='冻结 memtable 不更新结构版本(点查读到旧值/漏数据)',
file=LSM,
old=""" // v0.8.0:**前台变化也算结构变化**。读路径的"结构版本一致"必须覆盖""",
new=""" this.structureVersion--; // [MUTATION] 抵消下面的自增
// v0.8.0:**前台变化也算结构变化**。读路径的"结构版本一致"必须覆盖""",
test=B6, pattern='并发 flush 与 compaction 交错时',
),
dict(
name='sstable 越界策略不统一(scanAll 另有一套实现:越界即整体放弃)',
file=SSTABLE,
old=""" scanAll(callback: (key: string, value: Record<string, unknown>) => void): void {
for (const [key, value] of this.iterEntries(0, this.indexEntries.length - 1)) {
callback(key, value);
}
}""",
new=""" scanAll(callback: (key: string, value: Record<string, unknown>) => void): void {
// [MUTATION] 修复前的状态:这里另有一份解析循环,越界策略与 iterEntries 不同
const lenSize = this.lenFieldSize();
for (let bi = 0; bi < this.indexEntries.length; bi++) {
const blockData = this.getBlockData(this.indexEntries[bi]);
if (!blockData) continue;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
const count = blockView.getUint32(0, false);
let offset = 4;
for (let i = 0; i < count; i++) {
if (offset + lenSize > blockData.byteLength) return;
const keyLen = blockView.getUint32(offset, false);
offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength) return;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
offset += keyLen;
const valLen = blockView.getUint32(offset, false);
offset += lenSize;
if (offset + valLen > blockData.byteLength) return;
const valBytes = blockData.slice(offset, offset + valLen);
offset += valLen;
try { callback(key, JSON.parse(new TextDecoder().decode(valBytes))); } catch { /* skip */ }
}
}
}""",
test=B6, pattern='三个读取路径对同一个损坏文件给出一致结论',
),
dict(
name='vacuum 硬编码返回 6(报告与事实无关)',
file=ENGINE,
old=""" let compactedLevels = 0;
const isBottom = (level: number): boolean => level === MAX_LSM_LEVELS - 1;""",
new=""" let compactedLevels = 6; // [MUTATION] 旧行为:与事实无关的数字
const isBottom = (level: number): boolean => level === MAX_LSM_LEVELS - 1;""",
test=B6, pattern='vacuum 返回',
),
dict(
name='close() 落盘失败后不释放后端/锁(无 try/finally',
file=ENGINE,
old=""" let failure: unknown = null;
try {
await this.flushAllLsms();""",
new=""" let failure: unknown = null;
if (true) { // [MUTATION] 旧行为:失败即中断,收尾逻辑不再执行
await this.flushAllLsms();
await this.bufferPool.flushAll();
await this.advanceWalCheckpoint();
await this.backend.close();
if (this.dbLock) { await this.dbLock.release(); this.dbLock = null; }
this.schemas.clear();
this.opened = false;
return;
}
try {
await this.flushAllLsms();""",
test=B6, pattern='close\\(\\) 在落盘失败时',
),
dict(
name='恢复报告不聚合 LSM 侧丢弃(数据丢失不可见)',
file=ENGINE,
old=""" for (const lsm of this.allLsms()) {
const r = lsm.getRecoveryReport();
for (const d of r.droppedSSTables) {
dropped.push({ namespace: r.namespace, ...d });
}
if (r.dataLossSuspected) dataLoss = true;
}""",
new=""" // [MUTATION] 不聚合 LSM 报告""",
test=B6, pattern='恢复报告聚合',
),
]
CURRENT = {'path': None, 'content': None}
def _restore_current(*_args):
"""收到信号时也要把源码恢复原状(否则中断会留下变异源码)。"""
if CURRENT['path'] and CURRENT['content'] is not None:
io.open(CURRENT['path'], 'w', encoding='utf-8').write(CURRENT['content'])
print(f"\n[restore] {CURRENT['path']} 已恢复", flush=True)
sys.exit(130)
for _sig in ('SIGINT', 'SIGTERM', 'SIGHUP'):
try:
signal.signal(getattr(signal, _sig), _restore_current)
except (AttributeError, ValueError): # pragma: no cover - 平台差异
pass
def run(cmd):
return subprocess.run(cmd, cwd=ROOT, shell=True, capture_output=True, text=True)
def main():
only = sys.argv[1] if len(sys.argv) > 1 else None
results = []
for m in MUTATIONS:
if only and only not in m['name']:
continue
path = os.path.join(ROOT, m['file'])
original = io.open(path, encoding='utf-8').read()
if m['old'] not in original:
results.append((m['name'], 'SKIP(锚点未找到)'))
print(f"[skip] {m['name']}: 锚点未找到", flush=True)
continue
mutated = original.replace(m['old'], m['new'], 1)
CURRENT['path'] = path
CURRENT['content'] = original
io.open(path, 'w', encoding='utf-8').write(mutated)
try:
# 注意:用 shlex.quote 而不是 repr —— repr 会把正则里的反斜杠再转义一次,
# 于是 jest 收到 `\\(` 这种模式、匹配不到任何用例,脚本会把"没跑用例"
# 误判成"测试仍然通过"。
cmd = (f"npx jest {shlex.quote(m['test'])} -t {shlex.quote(m['pattern'])} "
f"--testPathIgnorePatterns='/node_modules/' 2>&1 | tail -80")
out = run(cmd).stdout
# 注意:不能只搜 "0 total" —— jest 的 "Snapshots: 0 total" 也会命中,
# 于是把"跑了用例且失败"误判成"用例未匹配"。
no_tests = bool(re.search(r'^Tests:\s+0 total', out, re.M)) or ('No tests found' in out)
failed = ('' in out) or ('Tests:' in out and 'failed' in out) or ('' in out)
if no_tests:
status = 'SKIP(用例未匹配)'
elif failed:
status = 'FAIL(如预期)'
else:
status = 'PASS(!!)'
results.append((m['name'], status))
msg = {
'FAIL(如预期)': '测试失败(变异被拦住)',
'PASS(!!)': '测试仍然通过!',
'SKIP(用例未匹配)': '用例未匹配(模式写错了)',
}[status]
print(f"[{'ok' if status == 'FAIL(如预期)' else 'BAD'}] {m['name']}{msg}", flush=True)
finally:
io.open(path, 'w', encoding='utf-8').write(original)
CURRENT['path'] = None
CURRENT['content'] = None
print('\n==== 变异验证汇总 ====')
bad = [r for r in results if r[1] != 'FAIL(如预期)']
for name, status in results:
print(f' {status:14s} {name}')
if bad:
print(f'\n{len(bad)} 条变异没有被测试拦住 —— 那些用例只是陪跑。')
return 1
print('\n全部变异都被对应用例拦住。')
return 0
if __name__ == '__main__':
sys.exit(main())
+671 -216
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+24 -4
View File
@@ -132,6 +132,19 @@ class MinHeap {
export class MergeIterator { export class MergeIterator {
private sources: EntrySource[]; private sources: EntrySource[];
private heap: MinHeap; private heap: MinHeap;
/**
* v0.8.0(B-6/47):上一次返回的条目所属来源 —— 它的下一条**推迟到下次 next()**
* 才拉取。
*
* 修复前是"弹出堆顶后立刻补充该来源的下一条",于是**消费者只取 N 条,
* 底层生成器却已经产出 N+1 条**(审计实测:limit=5 的流式扫描多算 1 条)。
* 在真惰性扫描里这不是纯性能问题:多拉的那一条会解析一整个 SSTable 块,
* 也让"提前终止时未消费部分不再解析"的宣称不完全成立。
*
* 延迟补充是安全的:单个来源内部 key 唯一且有序,因此被弹出条目的后续
* key 必然大于当前 key,不可能参与本次的重复 key 归并。
*/
private pendingRefill: number | null = null;
constructor() { constructor() {
this.sources = []; this.sources = [];
@@ -146,16 +159,20 @@ export class MergeIterator {
/** 获取下一个归并后的条目 */ /** 获取下一个归并后的条目 */
next(): [string, Record<string, unknown>] | null { next(): [string, Record<string, unknown>] | null {
// 上一轮被延迟的补充:现在才真正拉取(见 pendingRefill 说明)
if (this.pendingRefill !== null) {
const sourceIndex = this.pendingRefill;
this.pendingRefill = null;
this.seedFromSource(sourceIndex);
}
if (this.heap.size === 0) return null; if (this.heap.size === 0) return null;
const first = this.heap.pop()!; const first = this.heap.pop()!;
const key = first.key; const key = first.key;
let best = first; let best = first;
// 刷新 first 来源的下一个值 // 跳过重复 key:在多个来源中保留 sourceIndex 最小(最新)的条目。
this.seedFromSource(first.sourceIndex); // 重复条目必须**立即**从各自来源补充(否则它们会永久占住堆顶)。
// 跳过重复 key:在多个来源中保留 sourceIndex 最小(最新)的条目
while (this.heap.peek() && this.heap.peek()!.key === key) { while (this.heap.peek() && this.heap.peek()!.key === key) {
const dup = this.heap.pop()!; const dup = this.heap.pop()!;
this.seedFromSource(dup.sourceIndex); this.seedFromSource(dup.sourceIndex);
@@ -164,6 +181,9 @@ export class MergeIterator {
} }
} }
// 胜出来源的补充推迟到下一次 next()(消费者只取 N 条 → 底层只产出 N 条)
this.pendingRefill = first.sourceIndex;
return [best.key, best.value]; return [best.key, best.value];
} }
+43 -72
View File
@@ -66,39 +66,9 @@ export class SSTableReader {
const blockIdx = this.locateBlock(targetKey); const blockIdx = this.locateBlock(targetKey);
if (blockIdx < 0) return null; if (blockIdx < 0) return null;
const entry = this.indexEntries[blockIdx]; for (const [key, value] of this.iterEntries(blockIdx, blockIdx)) {
const blockData = this.getBlockData(entry); if (key === targetKey) return value;
// v0.4.1-fix: 残缺文件(meta 偏移超出实际长度)跳过该块,而非抛 RangeError
if (!blockData) return null;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
const lenSize = this.lenFieldSize();
const entryCount = blockView.getUint32(0, false);
let offset = 4;
// 顺序扫描 block 内的条目(生产中应二分查找)
for (let i = 0; i < entryCount; i++) {
if (offset + lenSize > blockData.byteLength) break;
const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength) break;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
offset += keyLen;
const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + valLen > blockData.byteLength) break;
const valBytes = blockData.slice(offset, offset + valLen);
offset += valLen;
if (key === targetKey) {
try {
return JSON.parse(new TextDecoder().decode(valBytes));
} catch {
return null;
}
}
} }
return null; return null;
} }
@@ -131,48 +101,49 @@ export class SSTableReader {
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1); const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1);
if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx) return; if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx) return;
const lenSize = this.lenFieldSize(); for (const [key, value] of this.iterEntries(startBlockIdx, endBlockIdx)) {
for (let bi = startBlockIdx; bi <= endBlockIdx && bi >= 0; bi++) { if (key >= startKey && key <= endKey) yield [key, value];
const entry = this.indexEntries[bi];
const blockData = this.getBlockData(entry);
// v0.4.1-fix: 残缺块跳过(继续后续块,不抛异常)
if (!blockData) continue;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
const blockEntryCount = blockView.getUint32(0, false);
let offset = 4;
for (let i = 0; i < blockEntryCount; i++) {
if (offset + lenSize > blockData.byteLength) break;
const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength) break;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
offset += keyLen;
const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + valLen > blockData.byteLength) break;
const valBytes = blockData.slice(offset, offset + valLen);
offset += valLen;
if (key >= startKey && key <= endKey) {
try {
const value = JSON.parse(new TextDecoder().decode(valBytes));
yield [key, value];
} catch {
// skip corrupted entry
}
}
}
} }
} }
/** 扫描所有条目 */ /** 扫描所有条目 */
scanAll(callback: (key: string, value: Record<string, unknown>) => void): void { scanAll(callback: (key: string, value: Record<string, unknown>) => void): void {
for (const [key, value] of this.iterEntries(0, this.indexEntries.length - 1)) {
callback(key, value);
}
}
// -----------------------------------------------------------------------
// 统一解析(v0.8.0
// -----------------------------------------------------------------------
/**
* v0.8.0B-6):**唯一一份**块内条目解析实现。
*
* 修复前这段解析被抄成三份(`get` / `scanLazy` / `scanAll`),并且三处的
* 越界策略不一致:点查与扫描对同一个损坏文件可能给出不同结论
*(审计:`sstable.ts:80-100/145-166/182-201`)。三份实现里只要有一处漏改,
* 就会重新出现"同一文件在不同路径下读出不同数据"。
*
* 现在的统一策略(对三个调用点完全一致):
* - 块缺失/越界 → 跳过该块,继续后续块(不抛异常);
* - 块内任一条目的长度字段越界 → 该块**剩余条目整体放弃**(截断块),继续后续块;
* - 条目的 JSON 解析失败 → 跳过该条目(视为不存在),不中断其他条目。
*
* @param fromBlock 起始块下标(含)
* @param toBlock 结束块下标(含)
*/
private *iterEntries(
fromBlock: number,
toBlock: number,
): Generator<[string, Record<string, unknown>]> {
const lenSize = this.lenFieldSize(); const lenSize = this.lenFieldSize();
for (const entry of this.indexEntries) { const start = Math.max(0, fromBlock);
const blockData = this.getBlockData(entry); const end = Math.min(toBlock, this.indexEntries.length - 1);
// v0.4.1-fix: 残缺块跳过(scanAll 继续后续块,不抛异常) const decoder = new TextDecoder();
for (let bi = start; bi <= end; bi++) {
const blockData = this.getBlockData(this.indexEntries[bi]);
if (!blockData) continue; if (!blockData) continue;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength); const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
@@ -184,7 +155,7 @@ export class SSTableReader {
const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize; offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength) break; if (offset + keyLen + lenSize > blockData.byteLength) break;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen)); const key = decoder.decode(blockData.slice(offset, offset + keyLen));
offset += keyLen; offset += keyLen;
const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false); const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize; offset += lenSize;
@@ -193,10 +164,10 @@ export class SSTableReader {
offset += valLen; offset += valLen;
try { try {
const value = JSON.parse(new TextDecoder().decode(valBytes)); const value = JSON.parse(decoder.decode(valBytes)) as Record<string, unknown>;
callback(key, value); yield [key, value];
} catch { } catch {
// skip corrupted entry // 损坏条目跳过(与三处调用点此前的"跳过损坏条目"策略一致)
} }
} }
} }
+42 -10
View File
@@ -24,7 +24,7 @@ export class FileManager implements PageIO {
} }
/** 初始化:从存储中读取元数据 */ /** 初始化:从存储中读取元数据 */
async init(dbName: string): Promise<void> { async init(dbName: string, watermarkFloor: number = 1): Promise<void> {
this.dbName = dbName; this.dbName = dbName;
const meta = await this.backend.read('__aria_meta'); const meta = await this.backend.read('__aria_meta');
let nextPageId = 1; let nextPageId = 1;
@@ -43,11 +43,30 @@ export class FileManager implements PageIO {
if (!Number.isNaN(id) && id + 1 > nextPageId) nextPageId = id + 1; if (!Number.isNaN(id) && id + 1 > nextPageId) nextPageId = id + 1;
} }
} }
this.nextPageId = nextPageId; // v0.8.0B-6):manifest 的 pageId 水位是**权威下限**(单调推进、永不复用),
if (!meta || !(meta instanceof ArrayBuffer) || meta.byteLength < 4 || nextPageId !== new DataView(meta as ArrayBuffer).getUint32(0, false)) { // 与"现存最大页面 id + 1"、"旧 __aria_meta" 三者取最大 —— 任何单一来源被
await this.saveMeta(); // 截断/回退都不会导致页面 id 复用。
if (Number.isFinite(watermarkFloor) && watermarkFloor > nextPageId) {
nextPageId = Math.floor(watermarkFloor);
} }
this.nextPageId = nextPageId;
this.metaLoaded = true; this.metaLoaded = true;
// v0.8.0B-6):`__aria_meta` 降级为**兼容/诊断提示**,不再作为提交点:
// 页面水位只由 manifest 提交(单一提交点),这里仅在旧值落后时补写一次,
// 且写入失败不影响引擎(旧版本读到的只是"落后但单调"的提示值)。
const legacyValue = meta && meta instanceof ArrayBuffer && meta.byteLength >= 4
? new DataView(meta as ArrayBuffer).getUint32(0, false)
: -1;
if (legacyValue !== nextPageId) {
try {
await this.writeLegacyWatermark(nextPageId);
} catch { /* 兼容提示写失败不影响正确性 */ }
}
}
/** v0.8.0: 当前页面 id 水位(下一个可分配 id)—— manifest 提交时记录 */
getNextPageId(): number {
return this.nextPageId;
} }
// ---- PageIO ---- // ---- PageIO ----
@@ -78,18 +97,18 @@ export class FileManager implements PageIO {
async allocatePageId(): Promise<number> { async allocatePageId(): Promise<number> {
const id = this.nextPageId++; const id = this.nextPageId++;
await this.saveMeta(); await this.persistWatermarkHint();
return id; return id;
} }
/** v0.4.5: 批量分配页面 ID(一次 meta 持久化,避免页面化 SSTable 保存时逐页写 meta */ /** v0.4.5: 批量分配页面 ID(一次提示写,避免页面化 SSTable 保存时逐页写 meta */
async allocatePageIds(count: number): Promise<number[]> { async allocatePageIds(count: number): Promise<number[]> {
if (count <= 0) return []; if (count <= 0) return [];
const ids: number[] = []; const ids: number[] = [];
const start = this.nextPageId; const start = this.nextPageId;
this.nextPageId += count; this.nextPageId += count;
for (let i = 0; i < count; i++) ids.push(start + i); for (let i = 0; i < count; i++) ids.push(start + i);
await this.saveMeta(); await this.persistWatermarkHint();
return ids; return ids;
} }
@@ -101,9 +120,20 @@ export class FileManager implements PageIO {
// ---- 辅助 ---- // ---- 辅助 ----
private async saveMeta(): Promise<void> { /**
* v0.8.0B-6):`__aria_meta` 只是**兼容提示**(旧版本/人工诊断用),
* 失败不抛错 —— 真正的提交点是 manifest 的 `pageIdWatermark`。
*/
private async persistWatermarkHint(): Promise<void> {
if (!this.metaLoaded) return;
try {
await this.writeLegacyWatermark(this.nextPageId);
} catch { /* 提示写失败不影响正确性(manifest 才是权威) */ }
}
private async writeLegacyWatermark(nextPageId: number): Promise<void> {
const buf = new ArrayBuffer(8); const buf = new ArrayBuffer(8);
new DataView(buf).setUint32(0, this.nextPageId, false); new DataView(buf).setUint32(0, nextPageId, false);
await this.backend.write('__aria_meta', buf); await this.backend.write('__aria_meta', buf);
} }
@@ -111,6 +141,8 @@ export class FileManager implements PageIO {
async clearAll(): Promise<void> { async clearAll(): Promise<void> {
await this.backend.clear(); await this.backend.clear();
this.nextPageId = 1; this.nextPageId = 1;
await this.saveMeta(); try {
await this.writeLegacyWatermark(1);
} catch { /* 提示写失败不影响正确性 */ }
} }
} }
+723
View File
@@ -0,0 +1,723 @@
/**
* AriaEngine Manifest — 存储层**单一提交点**
* @module engine/aria/store/manifest
*
* v0.8.0B-6):把原先"四处独立落盘、靠推理保持一致"的元状态收敛成
* **一份带 CRC 的原子提交记录**:
*
* ```
* ┌──────────────────────────────────────────────────────────────┐
* │ 数据落盘(SSTable 页面/文件) │
* │ ↓ │
* │ __aria_manifest_<generation> 提交(单文件 COW 原子写 + CRC) │
* │ ↓ │
* │ 才允许截断 WAL / 删除旧分片 │
* └──────────────────────────────────────────────────────────────┘
* ```
*
* 修复前的问题(全部是本模块要消除的根因 4 类):
* - `__aria_lsm_meta*` 是**裸 JSON**`JSON.parse` 失败时 `readMetaList()` 返回
* `[]` —— 损坏的元数据 = **静默空库**,随后 `repair()` 的孤儿页面清理会
* 把"没人引用"的活页全部删掉(不可逆);
* - 页面 id 水位(`__aria_meta`)、WAL 起始位置、各命名空间 meta 各自独立落盘,
* 崩溃窗口内三者可以互相矛盾;
* - WAL 截断只看内存状态:**没有任何持久记录**能证明"被截断的记录已落盘";
* - 陈旧实例(多标签页/多实例)可以直接覆盖新一代的 meta。
*
* 本模块的语义约定:
* 1. **只认最后一份 CRC 通过的世代**;若存在 manifest 文件但全部世代都无效,
* `load()` **抛错**而不是返回空状态(宁可打不开,也不能静默当空库);
* 2. 提交是**先写后验**:写完立刻回读校验(CRC + 世代号一致)才认为提交成功;
* 3. 至少保留**两代**(当前 + 上一代)用于回退;只有当回退窗口安全时才删除更早的世代;
* 4. 任何"引用不到的东西"在**恢复路径**上一律不删除(删除只发生在显式 repair /
* 已经过提交点确认的 compaction 之后);
* 5. 所有计数器(pageId 水位、各命名空间 SSTable id)**单调推进、永不复用**。
*/
import type { ColumnDef } from '../../../constants';
import { DatabaseError } from '../../../constants';
import type { SSTableMeta } from '../types';
import type { IStorageBackend } from './backend';
import { crc32 } from '../crc32';
// ---------------------------------------------------------------------------
// 常量
// ---------------------------------------------------------------------------
/** manifest 文件 key 前缀(世代号 8 位十进制) */
export const MANIFEST_KEY_PREFIX = '__aria_manifest_';
/** 魔数 'M' 'S' 'M' 'F'MetonaSqlark Manifest */
export const MANIFEST_MAGIC = 0x4d534d46;
/** 当前格式版本 */
export const MANIFEST_FORMAT_VERSION = 1;
/** 头部字节数:magic(4) + version(2) + headerSize(2) + generation(4) + payloadLen(4) + payloadCrc(4) + headerCrc(4) */
export const MANIFEST_HEADER_SIZE = 24;
/** 至少保留的世代数(当前 + 上一代) */
export const MANIFEST_RETAIN_GENERATIONS = 2;
/**
* 认领所有权时一次跨过的世代数(v0.8.0)。
*
* 为什么需要"跨一段"而不是简单地 +1:多实例(浏览器多标签页 / 无 Web Locks 环境)
* 下,旧实例的可能**已经在途**的提交会落在 +1 这个号上,从而覆盖新实例刚认领的
* 世代 —— 新实例随后的提交就会看到"别人的更高世代"而被判成陈旧实例,
* 两个实例互相拒绝(实测:第二个打开者 open 直接失败)。
* 一次跨过一段之后,旧实例的在途提交落在更低的号上,既不会覆盖认领,
* 也会在下一次提交时被正常拒绝。
*/
export const MANIFEST_TAKEOVER_STRIDE = 1000;
const MANIFEST_KEY_REGEX = /^__aria_manifest_(\d{8})$/;
// ---------------------------------------------------------------------------
// 类型
// ---------------------------------------------------------------------------
/** WAL 的权威位置状态(由 manifest 提交时写入) */
export interface ManifestWalState {
/**
* 仍需保留的最小分片序号。小于它的分片整体已落盘,可安全删除
* (删除失败也只是残留文件,恢复时按 `startSegment` 忽略)。
*/
startSegment: number;
/**
* 落盘水位(LSN 语义):`lsn <= startLsn` 的记录**已确认存在于已提交的
* SSTable 中**,恢复时可跳过;`startLsn` 只允许在"所有 LSM 均无未落盘数据"
* 的提交点上推进 —— 这是"截断 WAL 前必须先提交 manifest"的可验证形式。
*/
startLsn: number;
/** LSN 高水位:重启后从此继续递增(LSN 全库单调,永不回退/复用) */
nextLsn: number;
}
/**
* 待落盘冻结表意图。
*
* 冻结表的数据此刻只存在于内存(+ WAL)。把它写进 manifest 有两个作用:
* 1. **阻止水位推进**`wal.startLsn` 不得越过 `lsnAtFreeze`,否则这些记录会被
* 当成"已落盘"而跳过(崩溃即永久丢数据);
* 2. **恢复期可校验**:重开后若 manifest 声称有冻结表,但 WAL 里既无对应记录、
* SSTable 里也无对应数据,说明"已确认写入"真的丢了 —— 此时必须报错,
* 而不是安静地少一批行。
*/
export interface ManifestFrozenIntent {
/** LSM 命名空间('main' / 'idx_<table>_<col>' */
ns: string;
/** LSM 内部单调递增的冻结表 id */
id: number;
/** 条目数 */
entryCount: number;
/** 最小 key */
minKey: string;
/** 最大 key */
maxKey: string;
/** 冻结时刻的 WAL LSN(该表内容全部来自这之后的记录) */
lsnAtFreeze: number;
}
/** 实例所有权(陈旧实例保护) */
export interface ManifestOwner {
/** 提交方实例 id */
instanceId: string;
/** 单调递增的提交世代(每次提交 +1) */
epoch: number;
/** 首次打开时间(ms */
openedAt: number;
}
/** 单个 LSM 命名空间的持久状态 */
export interface ManifestNamespaceState {
/** 下一个 SSTable id(单调推进,永不复用) */
nextSstableId: number;
/** 该命名空间当前全部 SSTable meta(权威列表) */
sstables: SSTableMeta[];
}
/** manifest 完整内容 */
export interface AriaManifest {
formatVersion: number;
/** 世代号(每次提交 +1,与文件名一致) */
generation: number;
/** 页面 id 水位(下一页可分配 id;单调推进,永不复用) */
pageIdWatermark: number;
/** 命名空间 → 状态 */
namespaces: Record<string, ManifestNamespaceState>;
/** 表结构(列定义,与旧 `__aria_schemas` 同形) */
schemas: Record<string, Record<string, ColumnDef>>;
/** WAL 权威位置 */
wal: ManifestWalState;
/** 待落盘冻结表意图 */
frozen: ManifestFrozenIntent[];
/** 所有权 */
owner: ManifestOwner;
/** 提交时间(ms,诊断用) */
committedAt: number;
}
/** 加载结果 */
export interface ManifestLoadResult {
/** 最新有效世代的内容;null = 全新库(没有任何 manifest 文件) */
manifest: AriaManifest | null;
/** 最新有效世代号(全新库为 0) */
generation: number;
/** 是否曾有损坏/不可解析的世代被跳过(保守:存在则禁止自动清理旧世代) */
hadInvalidGenerations: boolean;
/** 跳过的世代及原因(诊断) */
skipped: { generation: number; reason: string }[];
}
/** 新建空 manifest 的参数 */
export interface EmptyManifestOptions {
instanceId: string;
now?: number;
}
// ---------------------------------------------------------------------------
// 空 manifest / 编解码
// ---------------------------------------------------------------------------
/** 创建一份空 manifest(全新库;或作为旧格式迁移的起点) */
export function createEmptyManifest(opts: EmptyManifestOptions): AriaManifest {
return {
formatVersion: MANIFEST_FORMAT_VERSION,
generation: 0,
pageIdWatermark: 1,
namespaces: {},
schemas: {},
wal: { startSegment: 0, startLsn: 0, nextLsn: 0 },
frozen: [],
owner: { instanceId: opts.instanceId, epoch: 0, openedAt: opts.now ?? Date.now() },
committedAt: opts.now ?? Date.now(),
};
}
/** manifest 文件 key */
export function manifestKey(generation: number): string {
return `${MANIFEST_KEY_PREFIX}${String(generation).padStart(8, '0')}`;
}
/** 从 key 解析世代号(非 manifest key 返回 null */
export function generationFromKey(key: string): number | null {
const m = MANIFEST_KEY_REGEX.exec(key);
return m ? Number(m[1]) : null;
}
/** 把 manifest 编码为字节(头部 + JSON 载荷) */
export function encodeManifest(manifest: AriaManifest): Uint8Array {
const payload = new TextEncoder().encode(JSON.stringify(serializeManifest(manifest)));
const buf = new ArrayBuffer(MANIFEST_HEADER_SIZE + payload.byteLength);
const view = new DataView(buf);
view.setUint32(0, MANIFEST_MAGIC, false);
view.setUint16(4, manifest.formatVersion, false);
view.setUint16(6, MANIFEST_HEADER_SIZE, false);
view.setUint32(8, manifest.generation >>> 0, false);
view.setUint32(12, payload.byteLength, false);
view.setUint32(16, payload.byteLength > 0 ? crc32(payload) : 0, false);
// 头部自身也带 CRCgeneration/长度被篡改时不会误判为有效世代)
view.setUint32(20, crc32(new Uint8Array(buf, 0, 20)), false);
new Uint8Array(buf, MANIFEST_HEADER_SIZE).set(payload);
return new Uint8Array(buf);
}
/** 解码结果:成功或明确的失败原因(绝不"失败当空库" */
export type ManifestDecodeResult =
| { ok: true; manifest: AriaManifest }
| { ok: false; reason: string };
/** 从字节解码 manifest(任何异常都转成结构化失败原因) */
export function decodeManifest(bytes: Uint8Array): ManifestDecodeResult {
try {
if (bytes.byteLength < MANIFEST_HEADER_SIZE) {
return { ok: false, reason: `too small (${bytes.byteLength} < ${MANIFEST_HEADER_SIZE})` };
}
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const magic = view.getUint32(0, false);
if (magic !== MANIFEST_MAGIC) {
return { ok: false, reason: `bad magic 0x${magic.toString(16)}` };
}
const headerCrc = view.getUint32(20, false);
const computedHeaderCrc = crc32(bytes.subarray(0, 20));
if (headerCrc !== computedHeaderCrc) {
return { ok: false, reason: `header CRC mismatch (stored=${headerCrc} computed=${computedHeaderCrc})` };
}
const version = view.getUint16(4, false);
if (version !== MANIFEST_FORMAT_VERSION) {
return { ok: false, reason: `unsupported format version ${version}` };
}
const headerSize = view.getUint16(6, false);
if (headerSize !== MANIFEST_HEADER_SIZE) {
return { ok: false, reason: `unexpected header size ${headerSize}` };
}
const generation = view.getUint32(8, false);
const payloadLength = view.getUint32(12, false);
if (payloadLength === 0 || payloadLength > bytes.byteLength - MANIFEST_HEADER_SIZE) {
return { ok: false, reason: `invalid payload length ${payloadLength}` };
}
const payload = bytes.subarray(MANIFEST_HEADER_SIZE, MANIFEST_HEADER_SIZE + payloadLength);
const payloadCrc = view.getUint32(16, false);
const computedPayloadCrc = crc32(payload);
if (payloadCrc !== computedPayloadCrc) {
return { ok: false, reason: `payload CRC mismatch (stored=${payloadCrc} computed=${computedPayloadCrc})` };
}
let parsed: unknown;
try {
parsed = JSON.parse(new TextDecoder().decode(payload));
} catch (error) {
return { ok: false, reason: `payload is not valid JSON: ${(error as Error).message}` };
}
const validated = validateManifestShape(parsed, generation);
if (!validated.ok) return validated;
return { ok: true, manifest: validated.manifest };
} catch (error) {
return { ok: false, reason: `decode threw: ${(error as Error).message}` };
}
}
// ---------------------------------------------------------------------------
// 序列化 / 校验
// ---------------------------------------------------------------------------
/** 序列化:只保留有意义的字段,并把 Uint8Array 类字段规整掉(manifest 必须是 JSON 可表示的) */
function serializeManifest(manifest: AriaManifest): AriaManifest {
const namespaces: Record<string, ManifestNamespaceState> = {};
for (const [ns, state] of Object.entries(manifest.namespaces)) {
namespaces[ns] = {
nextSstableId: state.nextSstableId,
sstables: state.sstables.map((m) => ({
id: m.id,
level: m.level,
minKey: m.minKey,
maxKey: m.maxKey,
blockCount: m.blockCount,
totalSize: m.totalSize,
// bloom 数据当前恒为 null;若被塞入 Uint8Array 则由 LSM 侧保证不为 manifest 内容
bloomData: null,
...(m.pageIds && m.pageIds.length > 0 ? { pageIds: [...m.pageIds] } : {}),
})),
};
}
return {
formatVersion: manifest.formatVersion,
generation: manifest.generation,
pageIdWatermark: manifest.pageIdWatermark,
namespaces,
schemas: manifest.schemas,
wal: {
startSegment: manifest.wal.startSegment,
startLsn: manifest.wal.startLsn,
nextLsn: manifest.wal.nextLsn,
},
frozen: manifest.frozen.map((f) => ({ ...f })),
owner: { ...manifest.owner },
committedAt: manifest.committedAt,
};
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function toNonNegativeInt(value: unknown, field: string): number {
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || !Number.isInteger(value)) {
throw new Error(`manifest field "${field}" must be a non-negative integer, got ${JSON.stringify(value)}`);
}
return value;
}
function toNonEmptyString(value: unknown, field: string): string {
if (typeof value !== 'string') {
throw new Error(`manifest field "${field}" must be a string, got ${JSON.stringify(value)}`);
}
return value;
}
/** 严格形状校验:任何不符合的类型都判该世代无效(而不是"部分采用" */
function validateManifestShape(value: unknown, generation: number): ManifestDecodeResult {
try {
if (!isPlainObject(value)) return { ok: false, reason: 'payload is not an object' };
const formatVersion = toNonNegativeInt(value.formatVersion, 'formatVersion');
if (formatVersion !== MANIFEST_FORMAT_VERSION) {
return { ok: false, reason: `payload formatVersion ${formatVersion}` };
}
const payloadGeneration = toNonNegativeInt(value.generation, 'generation');
if (payloadGeneration !== generation) {
return { ok: false, reason: `generation mismatch (header=${generation} payload=${payloadGeneration})` };
}
const pageIdWatermark = toNonNegativeInt(value.pageIdWatermark, 'pageIdWatermark');
// ---- namespaces ----
if (!isPlainObject(value.namespaces)) {
return { ok: false, reason: 'namespaces is not an object' };
}
const namespaces: Record<string, ManifestNamespaceState> = {};
for (const [ns, raw] of Object.entries(value.namespaces)) {
if (!isPlainObject(raw)) return { ok: false, reason: `namespace "${ns}" is not an object` };
const nextSstableId = toNonNegativeInt(raw.nextSstableId, `namespaces.${ns}.nextSstableId`);
if (!Array.isArray(raw.sstables)) {
return { ok: false, reason: `namespaces.${ns}.sstables is not an array` };
}
const sstables: SSTableMeta[] = [];
for (const item of raw.sstables) {
if (!isPlainObject(item)) return { ok: false, reason: `namespace "${ns}" has a non-object sstable` };
const id = toNonNegativeInt(item.id, `namespaces.${ns}.sstables[].id`);
const level = toNonNegativeInt(item.level, `namespaces.${ns}.sstables[].level`);
const blockCount = toNonNegativeInt(item.blockCount, `namespaces.${ns}.sstables[].blockCount`);
const totalSize = toNonNegativeInt(item.totalSize, `namespaces.${ns}.sstables[].totalSize`);
const minKey = toNonEmptyString(item.minKey, `namespaces.${ns}.sstables[].minKey`);
const maxKey = toNonEmptyString(item.maxKey, `namespaces.${ns}.sstables[].maxKey`);
let pageIds: number[] | undefined;
if (item.pageIds !== undefined) {
if (!Array.isArray(item.pageIds)) {
return { ok: false, reason: `namespaces.${ns}.sstables[].pageIds is not an array` };
}
pageIds = item.pageIds.map((pid, i) =>
toNonNegativeInt(pid, `namespaces.${ns}.sstables[].pageIds[${i}]`));
}
sstables.push({
id,
level,
minKey,
maxKey,
blockCount,
totalSize,
bloomData: null,
...(pageIds ? { pageIds } : {}),
});
}
namespaces[ns] = { nextSstableId, sstables };
}
// ---- schemas ----
if (!isPlainObject(value.schemas)) {
return { ok: false, reason: 'schemas is not an object' };
}
const schemas: Record<string, Record<string, ColumnDef>> = {};
for (const [table, cols] of Object.entries(value.schemas)) {
if (!isPlainObject(cols)) return { ok: false, reason: `schemas.${table} is not an object` };
schemas[table] = cols as Record<string, ColumnDef>;
}
// ---- wal ----
if (!isPlainObject(value.wal)) return { ok: false, reason: 'wal is not an object' };
const wal: ManifestWalState = {
startSegment: toNonNegativeInt(value.wal.startSegment, 'wal.startSegment'),
startLsn: toNonNegativeInt(value.wal.startLsn, 'wal.startLsn'),
nextLsn: toNonNegativeInt(value.wal.nextLsn, 'wal.nextLsn'),
};
if (wal.startLsn > wal.nextLsn) {
return { ok: false, reason: `wal.startLsn ${wal.startLsn} > wal.nextLsn ${wal.nextLsn}` };
}
// ---- frozen ----
if (!Array.isArray(value.frozen)) return { ok: false, reason: 'frozen is not an array' };
const frozen: ManifestFrozenIntent[] = [];
for (const item of value.frozen) {
if (!isPlainObject(item)) return { ok: false, reason: 'frozen has a non-object entry' };
frozen.push({
ns: toNonEmptyString(item.ns, 'frozen[].ns'),
id: toNonNegativeInt(item.id, 'frozen[].id'),
entryCount: toNonNegativeInt(item.entryCount, 'frozen[].entryCount'),
minKey: toNonEmptyString(item.minKey, 'frozen[].minKey'),
maxKey: toNonEmptyString(item.maxKey, 'frozen[].maxKey'),
lsnAtFreeze: toNonNegativeInt(item.lsnAtFreeze, 'frozen[].lsnAtFreeze'),
});
}
// ---- owner ----
if (!isPlainObject(value.owner)) return { ok: false, reason: 'owner is not an object' };
const owner: ManifestOwner = {
instanceId: toNonEmptyString(value.owner.instanceId, 'owner.instanceId'),
epoch: toNonNegativeInt(value.owner.epoch, 'owner.epoch'),
openedAt: toNonNegativeInt(value.owner.openedAt, 'owner.openedAt'),
};
const committedAt = toNonNegativeInt(value.committedAt, 'committedAt');
return {
ok: true,
manifest: {
formatVersion,
generation,
pageIdWatermark,
namespaces,
schemas,
wal,
frozen,
owner,
committedAt,
},
};
} catch (error) {
return { ok: false, reason: (error as Error).message };
}
}
// ---------------------------------------------------------------------------
// ManifestStore
// ---------------------------------------------------------------------------
export interface ManifestStoreOptions {
backend: IStorageBackend;
/** 本实例 id(陈旧实例保护用;默认随机生成) */
instanceId?: string;
/** 时间源(测试可注入) */
now?: () => number;
}
let instanceCounter = 0;
/**
* manifest 读写器:加载最新有效世代、串行提交、维护回退窗口。
*
* 注意:本类**不做数据落盘**,它只负责"提交点"。调用方必须先让数据真实落盘
* SSTable 页面 flush 完成),再 `commit()`。
*/
export class ManifestStore {
private backend: IStorageBackend;
private now: () => number;
private state: AriaManifest;
private loaded = false;
private hadInvalidGenerations = false;
private skipped: { generation: number; reason: string }[] = [];
/** 提交串行链:把 manifest 写入序列化成严格顺序 */
private commitChain: Promise<void> = Promise.resolve();
/** 已提交的世代号(= 最新有效世代) */
private generation = 0;
constructor(opts: ManifestStoreOptions) {
this.backend = opts.backend;
this.now = opts.now ?? (() => Date.now());
this.state = createEmptyManifest({
instanceId: opts.instanceId ?? `aria-${++instanceCounter}-${Math.random().toString(36).slice(2, 10)}`,
now: this.now(),
});
}
/** 本实例 id */
get instanceId(): string {
return this.state.owner.instanceId;
}
/** 当前内存态(调用方可直接修改;提交通过 `commit()` 串行化) */
get current(): AriaManifest {
return this.state;
}
/** 最近一次加载/提交后的世代号 */
get currentGeneration(): number {
return this.generation;
}
/**
* 加载最新有效世代。
*
* - 没有任何 manifest 文件 → `{ manifest: null }`(全新库);
* - 有文件且最新世代有效 → 采用它(更早的损坏世代只记录、不删除);
* - 有文件但**全部世代无效** → 抛 `ARIA_MANIFEST_CORRUPT`
* (绝不返回空状态:那会让上层把"元数据全坏"当成"空库"
* 随后 repair 还会把没人引用的活页删干净 —— 不可逆)。
*/
async load(): Promise<ManifestLoadResult> {
const keys = await this.backend.listKeys();
const generations = keys
.map((k) => generationFromKey(k))
.filter((g): g is number => g !== null)
.sort((a, b) => b - a);
const skipped: { generation: number; reason: string }[] = [];
for (const gen of generations) {
const raw = await this.backend.read(manifestKey(gen));
if (!raw) {
skipped.push({ generation: gen, reason: 'manifest file disappeared during load' });
continue;
}
const decoded = decodeManifest(new Uint8Array(raw));
if (!decoded.ok) {
skipped.push({ generation: gen, reason: decoded.reason });
continue;
}
if (decoded.manifest.generation !== gen) {
skipped.push({
generation: gen,
reason: `file name generation ${gen} != payload generation ${decoded.manifest.generation}`,
});
continue;
}
this.state = decoded.manifest;
this.generation = gen;
this.skipped = skipped;
this.hadInvalidGenerations = skipped.length > 0;
this.loaded = true;
return { manifest: decoded.manifest, generation: gen, hadInvalidGenerations: this.hadInvalidGenerations, skipped };
}
if (generations.length > 0) {
// 文件存在但一代都读不出来:这是元数据损坏,不是空库
throw new DatabaseError(
`AriaEngine manifest is corrupt: ${generations.length} generation(s) present, none passed validation ` +
`(${skipped.map((s) => `gen ${s.generation}: ${s.reason}`).join('; ')})`,
'ARIA_MANIFEST_CORRUPT',
);
}
this.loaded = true;
return { manifest: null, generation: 0, hadInvalidGenerations: false, skipped: [] };
}
/** 用一份外部状态替换内存态(旧格式迁移 / 测试用),不落盘 */
adopt(manifest: AriaManifest): void {
this.state = manifest;
this.generation = manifest.generation;
}
/**
* 认领所有权:把当前内存态提交为新世代(owner 换成本实例)。
*
* 与普通 `commit()` 的区别:这里**允许**接手别人提交的世代
* (多标签页的互斥由 Web Locks 负责;即便没有 Web Locks,接手者也只会让
* 旧实例后续提交被拒绝,而不是让旧实例静默覆盖新数据)。
*/
async claimOwnership(): Promise<AriaManifest> {
return this.commitInternal({ allowTakeover: true });
}
/**
* 提交当前内存态为新世代。
*
* 提交前会**重新读取磁盘上的最新世代号**:若它已经超过本实例上次提交的世代,
* 说明另一个实例在我们不知情的情况下提交过(陈旧实例)—— 此时抛
* `STALE_INSTANCE`,而不是用陈旧的内存态覆盖新一代(修复前 KVStore 快照被
* 静默覆盖、Aria 侧无任何保护)。
*/
async commit(): Promise<AriaManifest> {
return this.commitInternal({ allowTakeover: false });
}
private async commitInternal(opts: { allowTakeover: boolean }): Promise<AriaManifest> {
let result!: AriaManifest;
let failure: unknown = null;
const run = this.commitChain.then(async () => {
try {
result = await this.doCommit(opts);
} catch (error) {
failure = error;
}
});
this.commitChain = run;
await run;
if (failure) throw failure;
return result;
}
private async doCommit(opts: { allowTakeover: boolean }): Promise<AriaManifest> {
if (!this.loaded) {
throw new DatabaseError('ManifestStore.commit() before load()', 'ARIA_MANIFEST_NOT_LOADED');
}
const onDisk = await this.newestGenerationOnDisk();
// 陈旧实例判定要**跳过已知损坏的世代**:损坏世代(撕裂写/介质坏块)不是
// "另一个实例提交的状态",把它算进来会让库永远无法再提交(实测:
// 一个损坏的更高世代把后续所有提交都拦成 STALE_INSTANCE)。
const knownInvalid = new Set(this.skipped.map((s) => s.generation));
let newestForeign = 0;
for (const gen of await this.listGenerationNumbers()) {
if (gen > this.generation && !knownInvalid.has(gen)) newestForeign = Math.max(newestForeign, gen);
}
if (!opts.allowTakeover && newestForeign > this.generation) {
throw new DatabaseError(
`Refusing to commit: another instance committed generation ${newestForeign} ` +
`(this instance last committed ${this.generation}) — stale instance`,
'STALE_INSTANCE',
);
}
// 但世代号必须大于**任何**已存在的文件(含损坏世代),否则会覆盖它。
// 认领(takeover)时一次跨过一段:见 MANIFEST_TAKEOVER_STRIDE 的说明。
const nextGeneration = opts.allowTakeover
? onDisk + MANIFEST_TAKEOVER_STRIDE
: onDisk + 1;
const next: AriaManifest = {
...this.state,
formatVersion: MANIFEST_FORMAT_VERSION,
generation: nextGeneration,
namespaces: this.state.namespaces,
schemas: this.state.schemas,
wal: { ...this.state.wal },
frozen: this.state.frozen.map((f) => ({ ...f })),
owner: {
instanceId: this.state.owner.instanceId,
epoch: this.state.owner.epoch + 1,
openedAt: this.state.owner.openedAt,
},
committedAt: this.now(),
};
const encoded = encodeManifest(next);
await this.backend.write(manifestKey(nextGeneration), encoded.buffer.slice(
encoded.byteOffset,
encoded.byteOffset + encoded.byteLength,
) as ArrayBuffer);
// 先写后验:回读并校验,只有真正可读回的新世代才算提交成功
const readBack = await this.backend.read(manifestKey(nextGeneration));
if (!readBack) {
throw new DatabaseError(
`Manifest commit verification failed: generation ${nextGeneration} not readable after write`,
'ARIA_MANIFEST_WRITE_FAILED',
);
}
const verified = decodeManifest(new Uint8Array(readBack));
if (!verified.ok || verified.manifest.generation !== nextGeneration) {
throw new DatabaseError(
`Manifest commit verification failed: generation ${nextGeneration} ` +
`(${verified.ok ? 'generation mismatch' : verified.reason})`,
'ARIA_MANIFEST_WRITE_FAILED',
);
}
this.state = next;
this.generation = nextGeneration;
// 回退窗口之外的历史世代:只有在**没有损坏世代**时才清理
//(存在损坏世代时保留全部,绝不因为"读不出来"就删掉可能是唯一副本的东西)
if (!this.hadInvalidGenerations) {
await this.pruneOldGenerations(nextGeneration);
}
return next;
}
/** 磁盘上最大的 manifest 世代号(乐观并发检查;只读 key 名单) */
async newestGenerationOnDisk(): Promise<number> {
return (await this.listGenerationNumbers()).reduce((max, g) => Math.max(max, g), 0);
}
/** 磁盘上全部 manifest 世代号(升序无关,仅用于判定) */
private async listGenerationNumbers(): Promise<number[]> {
const keys = await this.backend.listKeys();
const out: number[] = [];
for (const k of keys) {
const gen = generationFromKey(k);
if (gen !== null) out.push(gen);
}
return out;
}
/** 删除回退窗口之外的世代(当前 + 上一代保留) */
private async pruneOldGenerations(currentGeneration: number): Promise<void> {
const keys = await this.backend.listKeys();
const stale = keys
.map((k) => generationFromKey(k))
.filter((g): g is number => g !== null && g < currentGeneration - (MANIFEST_RETAIN_GENERATIONS - 1));
if (stale.length === 0) return;
try {
await this.backend.deleteMany(stale.map((g) => manifestKey(g)));
} catch {
// 删除失败只是残留文件(下次提交再清),不影响正确性
}
}
}
+51 -12
View File
@@ -23,6 +23,18 @@ import { compressLZ4, decompressLZ4 } from '../compression/lz4';
export class PageSSTableStore { export class PageSSTableStore {
/** SSTable id → 页面 ID 列表(save 时记录,saveMeta 时注入 meta */ /** SSTable id → 页面 ID 列表(save 时记录,saveMeta 时注入 meta */
private pageIds = new Map<number, number[]>(); private pageIds = new Map<number, number[]>();
/**
* v0.8.0B-6):已退休(被 compaction 取代 / 已被 manifest 摘除)但仍可能有
* 在途读者持有引用的 SSTable → 页面 ID 列表。
*
* 为什么必须保留:读路径是"先取 meta 快照、再按 id 加载数据",快照与加载之间
* 可以插入一次 compaction。若退休时立刻忘掉 pageIds,在途读者的 `load()` 就
* 找不到页面,只能把"文件已退休"误判为"数据缺失"(旧代码会顺手删掉 meta 并
* 打一条损坏告警)。保留到物理删除为止,语义才是自洽的。
*/
private retiredPageIds = new Map<number, number[]>();
/** SSTable id → 落盘字节数(load 时截断最后一页 0 填充) */
private storedSizes = new Map<number, number>();
constructor( constructor(
private fileManager: FileManager, private fileManager: FileManager,
@@ -72,24 +84,47 @@ export class PageSSTableStore {
this.bufferPool.unpin(page); this.bufferPool.unpin(page);
} }
this.pageIds.set(id, ids); this.pageIds.set(id, ids);
this.retiredPageIds.delete(id);
this.storedSizes.set(id, payload.byteLength);
return { storedSize: payload.byteLength }; return { storedSize: payload.byteLength };
} }
/** 获取指定 SSTable 的页面 ID 列表(saveMeta 注入用) */ /** 获取指定 SSTable 的页面 ID 列表(saveMeta 注入用;未注册返回 undefined */
getPageIds(id: number): number[] | undefined { getPageIds(id: number): number[] | undefined {
return this.pageIds.get(id); return this.pageIds.get(id) ?? this.retiredPageIds.get(id);
}
/** v0.8.0:打开时从 manifest 把已有 SSTable 的页面映射注册进来(load 不再依赖调用方传参) */
registerPageIds(id: number, ids: number[], storedSize?: number): void {
if (this.pageIds.has(id)) return;
this.pageIds.set(id, [...ids]);
if (typeof storedSize === 'number') this.storedSizes.set(id, storedSize);
}
/**
* v0.8.0:标记某 SSTable 已退休(被合并产物取代)。
* 页面映射保留,在途读者仍能读到旧数据;物理删除由 `delete()` 完成。
*/
retirePageIds(id: number): void {
const ids = this.pageIds.get(id);
if (!ids) return;
this.pageIds.delete(id);
this.retiredPageIds.set(id, ids);
} }
/** /**
* 按页面 ID 列表读取并拼接为完整字节流。 * 按页面 ID 列表读取并拼接为完整字节流。
* @param totalSize 页面中**实际存储**的字节数(`save()` 返回的 storedSize * @param pageIds 页面 ID 列表(缺省时用内部注册的映射)
* 即压缩后长度)——最后一页可能有 0 填充,按它截断。 * @param totalSize 页面中**实际存储**的字节数(缺省时用内部记录)
* @returns 缺失页面/读取失败返回 null(调用方视为损坏并清理) * @returns 缺失页面/读取失败返回 null(调用方视为损坏并清理)
*/ */
async load(id: number, pageIds: number[], totalSize: number): Promise<Uint8Array | null> { async load(id: number, pageIds?: number[], totalSize?: number): Promise<Uint8Array | null> {
if (pageIds.length === 0) return null; const ids = pageIds ?? this.pageIds.get(id) ?? this.retiredPageIds.get(id);
if (!ids || ids.length === 0) return null;
const size = totalSize ?? this.storedSizes.get(id);
if (typeof size !== 'number') return null;
const chunks: Uint8Array[] = []; const chunks: Uint8Array[] = [];
for (const pageId of pageIds) { for (const pageId of ids) {
const page = await this.bufferPool.getPage(pageId); const page = await this.bufferPool.getPage(pageId);
if (!page) return null; if (!page) return null;
// 立即复制(后续驱逐安全) // 立即复制(后续驱逐安全)
@@ -97,7 +132,7 @@ export class PageSSTableStore {
this.bufferPool.unpin(page); this.bufferPool.unpin(page);
} }
const total = chunks.reduce((s, c) => s + c.byteLength, 0); const total = chunks.reduce((s, c) => s + c.byteLength, 0);
const out = new Uint8Array(Math.min(total, totalSize)); const out = new Uint8Array(Math.min(total, size));
let off = 0; let off = 0;
for (const c of chunks) { for (const c of chunks) {
const take = Math.min(c.byteLength, out.byteLength - off); const take = Math.min(c.byteLength, out.byteLength - off);
@@ -105,19 +140,23 @@ export class PageSSTableStore {
out.set(c.subarray(0, take), off); out.set(c.subarray(0, take), off);
off += take; off += take;
} }
this.pageIds.delete(id); // v0.8.0**不再**在这里丢掉 pageIds —— 退休 SSTable 的在途读者仍会调用 load,
// 丢掉映射会让它们把"已退休"误判成"数据缺失"。物理删除由 delete() 负责。
// v0.8.0A38):解压(与 save 的加密/压缩顺序对称) // v0.8.0A38):解压(与 save 的加密/压缩顺序对称)
return this.compression ? decompressLZ4(out) : out; return this.compression ? decompressLZ4(out) : out;
} }
/** 释放页面(删除物理页面文件 + 移出 BufferPool */ /** 释放页面(删除物理页面文件 + 移出 BufferPool;同时清掉活跃与退休映射 */
async delete(id: number, pageIds: number[]): Promise<void> { async delete(id: number, pageIds?: number[]): Promise<void> {
for (const pageId of pageIds) { const ids = pageIds ?? this.pageIds.get(id) ?? this.retiredPageIds.get(id) ?? [];
for (const pageId of ids) {
this.bufferPool.removePage(pageId); this.bufferPool.removePage(pageId);
try { try {
await this.fileManager.freePageId(pageId); await this.fileManager.freePageId(pageId);
} catch { /* 清理失败不阻塞 */ } } catch { /* 清理失败不阻塞 */ }
} }
this.pageIds.delete(id); this.pageIds.delete(id);
this.retiredPageIds.delete(id);
this.storedSizes.delete(id);
} }
} }
+21 -3
View File
@@ -12,6 +12,18 @@ import type { WAL } from './log';
export interface Flushable { export interface Flushable {
flushAll(): Promise<void>; flushAll(): Promise<void>;
/**
* v0.8.0B-6/55):**只落 memtable**(不等待 compaction)。
*
* 为什么必须分开:checkpoint 要保证的是"WAL 覆盖的数据已落盘",而 compaction
* 只是重排**已经落盘**的 SSTable。修复前 checkpoint 会先 `lsm.flush()` 再
* `flushable.flushAll()`,两者都会排空后台维护链 —— 后台 compaction 跑几秒,
* 写路径每 1000 次操作就要等它一次(v0.6.1 记录的 "8~11s 悬崖"的另一半)。
*
* 未实现该方法的调用方退回旧语义(先 flush LSM 再 flushAll),
* 因此这个接口对既有测试替身保持兼容。
*/
flushMemtables?(): Promise<void>;
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -60,9 +72,15 @@ export class CheckpointManager {
} }
async checkpoint(): Promise<void> { async checkpoint(): Promise<void> {
await this.lsm.flush(); if (this.flushable && typeof this.flushable.flushMemtables === 'function') {
if (this.flushable) { // v0.8.0B-6/55):只等 memtable 落盘;compaction 继续在后台跑
await this.flushable.flushAll(); await this.flushable.flushMemtables();
} else {
// 兼容路径(测试替身 / 未实现新接口的调用方):旧语义
await this.lsm.flush();
if (this.flushable) {
await this.flushable.flushAll();
}
} }
await this.wal.checkpoint(); await this.wal.checkpoint();
this.opCount = 0; this.opCount = 0;
+162 -11
View File
@@ -20,8 +20,10 @@
* └──────────┴──────────────┴──────────┘ * └──────────┴──────────────┴──────────┘
*/ */
import { DatabaseError } from '../../../constants';
import { WALRecordType, type WALRecord } from '../types'; import { WALRecordType, type WALRecord } from '../types';
import { crc32 } from '../crc32'; import { crc32 } from '../crc32';
import type { WALReadResult } from './segmented_store';
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// WAL 存储接口 // WAL 存储接口
@@ -36,6 +38,61 @@ export interface WALStore {
truncate(): Promise<void>; truncate(): Promise<void>;
/** 检查 WAL 是否存在 */ /** 检查 WAL 是否存在 */
exists(): Promise<boolean>; exists(): Promise<boolean>;
/**
* v0.8.0(可选,分片存储实现):从指定分片起读取,并把空洞如实返回。
* 未实现的 store 由 WAL 回退为 `readAll()`(无空洞信息)。
*/
readAllFrom?(fromSegment: number): Promise<WALReadResult>;
/**
* v0.8.0(可选,分片存储实现):删除整体已落盘的前缀分片,
* 返回仍需保留的最小分片号。未实现时 WAL 只在"全部已落盘"时整体截断。
*
* @param latestLsn 当前 LSN 高水位(判断"最后一个分片"是否也被完整覆盖)
*/
truncateBefore?(durableLsn: number, latestLsn?: number): Promise<number>;
/**
* v0.8.0(可选):整库清空后重置分片编号(仅 `clearAll` 这类"介质整体抹掉、
* manifest 也从 0 开始"的场景)。
*/
reset?(): void;
/**
* v0.8.0(可选,分片存储实现):**只计算**仍需保留的最小分片号(无副作用)。
* 调用方必须先把它提交进 manifest,再调用 `truncateBefore` 删除 —— 顺序反了
* 会让恢复无法区分"正常清理过的前缀"与"介质丢了一段记录"。
*/
planKeepFrom?(durableLsn: number, latestLsn?: number): Promise<number>;
}
/** WAL 恢复的边界选项(v0.8.0 */
export interface WALRecoverOptions {
/**
* 跳过 `lsn <= fromLsn` 的记录。
*
* 这些记录已由 manifest 证明存在于已提交的 SSTable 中(`wal.startLsn`),
* 重放它们不仅浪费启动时间,还会把**已经删除的数据复活**
*(旧记录晚于 tombstone 重放时)。
*/
fromLsn?: number;
/** 从该分片号开始读取(manifest 的 `wal.startSegment` */
fromSegment?: number;
/**
* 允许活跃区间内的分片空洞。默认 false → 抛 `ARIA_WAL_GAP`。
*
* 为什么默认抛错:空洞意味着"某个已提交事务的记录缺失",静默继续会把
* 不完整的状态当成完整状态。引擎显式传 `allowGaps: true` 并把它写进
* 恢复报告(用户可见),而不是让调用方悄悄少一批数据。
*/
allowGaps?: boolean;
}
/** 最近一次恢复的诊断信息 */
export interface WALRecoveryInfo {
applied: number;
skipped: number;
maxLsn: number;
gaps: number[];
fromSegment: number;
fromLsn: number;
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -50,6 +107,8 @@ export class WAL {
private syncMode: 'full' | 'batch' | 'none'; private syncMode: 'full' | 'batch' | 'none';
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数(full/batch/none 通用) */ /** v0.3.3: 未 checkpoint 的 WAL 累计字节数(full/batch/none 通用) */
private bufferedBytes = 0; private bufferedBytes = 0;
/** v0.8.0: 最近一次恢复诊断 */
private lastRecoveryInfo: WALRecoveryInfo | null = null;
constructor(store: WALStore, enabled: boolean = true, syncMode: 'full' | 'batch' | 'none' = 'batch') { constructor(store: WALStore, enabled: boolean = true, syncMode: 'full' | 'batch' | 'none' = 'batch') {
this.store = store; this.store = store;
@@ -137,22 +196,72 @@ export class WAL {
/** 从 WAL 恢复未提交的事务数据 */ /** 从 WAL 恢复未提交的事务数据 */
async recover( async recover(
applyRecord: (record: WALRecord) => void, applyRecord: (record: WALRecord) => void,
opts: WALRecoverOptions = {},
): Promise<number> { ): Promise<number> {
if (!this.enabled) return 0; if (!this.enabled) return 0;
const exists = await this.store.exists(); const fromSegment = opts.fromSegment ?? 0;
if (!exists) return 0; const fromLsn = opts.fromLsn ?? 0;
let data: Uint8Array;
let gaps: number[] = [];
const data = await this.store.readAll(); if (typeof this.store.readAllFrom === 'function') {
if (data.byteLength === 0) return 0; const result = await this.store.readAllFrom(fromSegment);
data = result.data;
const records = this.decodeAllRecords(data); gaps = result.gaps;
for (const record of records) { } else {
applyRecord(record); const exists = await this.store.exists();
if (!exists) return 0;
data = await this.store.readAll();
} }
this.lsn = records.length > 0 ? records[records.length - 1].lsn : 0; if (gaps.length > 0 && !opts.allowGaps) {
return records.length; throw new DatabaseError(
`WAL segment gap detected in live range (missing segment(s): ${gaps.join(', ')}) — ` +
'records after the gap cannot be verified; refusing to continue silently',
'ARIA_WAL_GAP',
);
}
const records = data.byteLength === 0 ? [] : this.decodeAllRecords(data);
let applied = 0;
let skipped = 0;
let maxLsn = 0;
for (const record of records) {
if (record.lsn > maxLsn) maxLsn = record.lsn;
// v0.8.0: lsn <= startLsn 的记录已由 manifest 证明落盘,跳过(不再重复重放)
if (record.lsn <= fromLsn) {
skipped++;
continue;
}
applyRecord(record);
applied++;
}
this.lsn = Math.max(this.lsn, maxLsn);
// 磁盘上的全部记录已经读出来了,缓冲区里不再有待落盘记录
this.buffer = [];
this.bufferedBytes = 0;
this.lastRecoveryInfo = { applied, skipped, maxLsn, gaps, fromSegment, fromLsn };
return applied;
}
/** v0.8.0: 最近一次恢复诊断(applied/skipped/gaps */
getLastRecoveryInfo(): WALRecoveryInfo | null {
return this.lastRecoveryInfo ? { ...this.lastRecoveryInfo, gaps: [...this.lastRecoveryInfo.gaps] } : null;
}
/** v0.8.0: 当前 LSN 高水位(manifest 记录它以保证重启后 LSN 继续单调) */
getLsn(): number {
return this.lsn;
}
/**
* v0.8.0: 从 manifest 的高水位继续 LSN(重启后 LSN 全库单调,永不复用)。
* 只允许上调,不允许回退。
*/
setLsn(lsn: number): void {
if (Number.isFinite(lsn) && lsn > this.lsn) this.lsn = lsn;
} }
// ======================================================================= // =======================================================================
@@ -164,8 +273,50 @@ export class WAL {
if (!this.enabled) return; if (!this.enabled) return;
await this.flush(); await this.flush();
await this.store.truncate(); await this.store.truncate();
this.lsn = 0;
this.bufferedBytes = 0; this.bufferedBytes = 0;
// v0.8.0: **不再把 lsn 归零**。LSN 是 manifest 记录的全库单调水位
// `wal.nextLsn`),归零会让"跳过 lsn <= startLsn"的判定与历史分片冲突:
// 同一段 LSN 区间会对应两批完全不同的记录。
}
/** v0.8.0:整库清空后重置分片编号(供 `clearAll` 使用) */
reset(): void {
if (typeof this.store.reset === 'function') this.store.reset();
}
/**
* v0.8.0B-6):查询"提交之后可以删到哪个分片"(无副作用)。
* @returns 仍需保留的最小分片号(未实现分片能力的 store 返回 0 = 全部保留)
*/
async planKeepFromSegment(durableLsn: number): Promise<number> {
if (!this.enabled) return 0;
if (typeof this.store.planKeepFrom === 'function') {
return this.store.planKeepFrom(durableLsn, this.lsn);
}
return 0;
}
/**
* v0.8.0B-6):**按落盘水位**截断 WAL —— manifest 提交之后的清理动作。
*
* 与 `checkpoint()` 的区别:这里只删除"整段记录都 <= durableLsn"的前缀分片,
* 因此可以安全地在**后台 compaction 仍在进行**时调用。
*
* @returns 仍需保留的最小分片号(调用方应写入 manifest 的 `wal.startSegment`
*/
async checkpointBefore(durableLsn: number): Promise<number> {
if (!this.enabled) return 0;
// 先把缓冲记录落盘:否则"边界分片"的判定会漏掉刚写进缓冲的记录
await this.flush();
if (typeof this.store.truncateBefore === 'function') {
return this.store.truncateBefore(durableLsn, this.lsn);
}
// 回退(无分片能力的 store):只有确认"全部记录都已落盘"才整体截断
if (durableLsn >= this.lsn) {
await this.store.truncate();
this.bufferedBytes = 0;
}
return 0;
} }
// ======================================================================= // =======================================================================
+238 -18
View File
@@ -16,7 +16,9 @@ import type { IStorageBackend } from '../store/backend';
/** 分片文件名:__wal_%06d.bin */ /** 分片文件名:__wal_%06d.bin */
export const WAL_SEGMENT_PREFIX = '__wal_'; export const WAL_SEGMENT_PREFIX = '__wal_';
const SEGMENT_REGEX = /^__wal_(\d{6})\.bin$/; // v0.8.0:分片号只增不减(见 truncateBefore),因此不能假设一定是 6 位 ——
// 超过 999999 之后若仍按 {6} 匹配,分片会突然"消失"(静默丢日志)。
const SEGMENT_REGEX = /^__wal_(\d{6,})\.bin$/;
const LEGACY_RECORD_REGEX = /^__wal_(\d+)$/; const LEGACY_RECORD_REGEX = /^__wal_(\d+)$/;
const LEGACY_COUNT_KEY = '__wal_count'; const LEGACY_COUNT_KEY = '__wal_count';
@@ -34,6 +36,24 @@ export interface WALStore {
exists(): Promise<boolean>; exists(): Promise<boolean>;
} }
/** 读取结果:数据 + 空洞诊断(v0.8.0) */
export interface WALReadResult {
/** 有效前缀数据(自 fromSegment 起、连续分片拼接) */
data: Uint8Array;
/** 起始分片号 */
fromSegment: number;
/** 实际读到的分片号(升序) */
segments: number[];
/**
* 活跃区间内的空洞(缺失的分片号)。非空意味着"这之后的记录不可信" ——
* v0.8.0 起由调用方(WAL/引擎)**显式上报**,不再静默丢弃尾部
* (修复前:序号不连续 → 直接把空洞之后的分片全部丢掉且没有任何提示)。
*/
gaps: number[];
/** 最后一条记录之后仍缺失的分片数(尾部截断诊断) */
missingTail: number;
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// SegmentedWALStore // SegmentedWALStore
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -44,6 +64,14 @@ export class SegmentedWALStore implements WALStore {
private currentSegment = 0; private currentSegment = 0;
/** 当前分片字节数(内存跟踪,append 切分片判断) */ /** 当前分片字节数(内存跟踪,append 切分片判断) */
private currentSize = 0; private currentSize = 0;
/**
* v0.8.0:分片 → 该分片**首条记录**的 LSN。
*
* 用途:manifest 提交后需要知道"哪些分片整体已落盘可以删除"。
* 记录格式里 LSN 是每条记录的第 1 个字段,因此取分片前 4 字节即可定位;
* 无需改变 WAL 二进制格式(旧库分片同样适用)。
*/
private segmentFirstLsn = new Map<number, number>();
constructor( constructor(
private backend: IStorageBackend, private backend: IStorageBackend,
@@ -64,9 +92,18 @@ export class SegmentedWALStore implements WALStore {
this.currentSegment++; this.currentSegment++;
this.currentSize = 0; this.currentSize = 0;
} }
const key = this.segmentKey(this.currentSegment); const seq = this.currentSegment;
const key = this.segmentKey(seq);
const copy = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer; const copy = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
// 记录该分片首条记录的 LSN(分片内记录按写入顺序追加,首条即最小 LSN)
if (!this.segmentFirstLsn.has(seq) && data.byteLength >= 4) {
this.segmentFirstLsn.set(
seq,
new DataView(data.buffer, data.byteOffset, 4).getUint32(0, false),
);
}
if (typeof this.backend.append === 'function') { if (typeof this.backend.append === 'function') {
await this.backend.append!(key, copy); await this.backend.append!(key, copy);
} else { } else {
@@ -84,22 +121,48 @@ export class SegmentedWALStore implements WALStore {
this.currentSize += data.byteLength; this.currentSize += data.byteLength;
} }
async readAll(): Promise<Uint8Array> { /** 列出存储上的分片(升序) */
private async listSegments(): Promise<{ seq: number; key: string }[]> {
const keys = await this.backend.listKeys(); const keys = await this.backend.listKeys();
return keys
// ---- 新格式分片 ----
const segments = keys
.filter((k) => SEGMENT_REGEX.test(k)) .filter((k) => SEGMENT_REGEX.test(k))
.map((k) => ({ seq: Number(k.match(SEGMENT_REGEX)![1]), key: k })) .map((k) => ({ seq: Number(k.match(SEGMENT_REGEX)![1]), key: k }))
.sort((a, b) => a.seq - b.seq); .sort((a, b) => a.seq - b.seq);
}
// 空洞检测:分片序号必须从 0 严格连续,空洞后的分片整体丢弃 /**
let keepCount = 0; * v0.8.0:从 `fromSegment` 开始读取,并把空洞**如实返回**(不再静默丢弃尾部)。
for (let i = 0; i < segments.length; i++) { *
if (segments[i].seq !== i) break; * 为什么必须报告空洞:`truncateBefore` 只会删除**前缀**分片,因此活跃区间内
keepCount = i + 1; * 出现空洞只可能来自介质损坏/外部删除 —— 此时空洞之后的记录无法确认是否属于
* 同一个连续历史。修复前 `readAll` 遇到空洞直接丢弃空洞之后的全部记录,
* 调用方(引擎恢复)完全无法感知"少了一批已提交事务"。
*/
async readAllFrom(fromSegment: number = 0): Promise<WALReadResult> {
const keys = await this.backend.listKeys();
const allSegments = await this.listSegments();
const segments = allSegments.filter((s) => s.seq >= fromSegment);
// 空洞检测:活跃区间内分片序号必须连续。
// v0.8.0:**前缀缺失**同样算空洞 —— 调用方(manifest 的 `wal.startSegment`
// 保证"小于 startSegment 的分片已整体落盘、可以不存在",因此活跃区间内
// 第一片就缺席(segments[0].seq > fromSegment)意味着那段记录不知所踪。
const gaps: number[] = [];
if (segments.length > 0) {
for (let missing = fromSegment; missing < segments[0].seq; missing++) gaps.push(missing);
}
for (let i = 1; i < segments.length; i++) {
for (let missing = segments[i - 1].seq + 1; missing < segments[i].seq; missing++) {
gaps.push(missing);
}
}
// 空洞之后的分片整体丢弃(长度字段链无法跨空洞验证),并计入 diagnostics
let kept = segments;
if (gaps.length > 0) {
const firstGap = gaps[0];
kept = segments.filter((s) => s.seq < firstGap);
} }
const validSegments = segments.slice(0, keepCount);
// ---- 旧格式兼容:__wal_N 单记录键(迁移前数据) ---- // ---- 旧格式兼容:__wal_N 单记录键(迁移前数据) ----
const legacyKeys = keys const legacyKeys = keys
@@ -114,15 +177,23 @@ export class SegmentedWALStore implements WALStore {
if (raw) parts.push(new Uint8Array(raw)); if (raw) parts.push(new Uint8Array(raw));
} }
// 新格式分片在后 // 新格式分片在后
for (const { key } of validSegments) { const readSegments: number[] = [];
for (const { seq, key } of kept) {
const raw = await this.backend.read(key); const raw = await this.backend.read(key);
if (raw) parts.push(new Uint8Array(raw)); if (!raw) continue;
const bytes = new Uint8Array(raw);
if (!this.segmentFirstLsn.has(seq) && bytes.byteLength >= 4) {
this.segmentFirstLsn.set(seq, new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0, false));
}
readSegments.push(seq);
parts.push(bytes);
} }
// 同步当前分片状态(追加定位) // 同步当前分片状态(追加定位)
if (validSegments.length > 0) { if (segments.length > 0) {
this.currentSegment = validSegments[validSegments.length - 1].seq; const lastSeq = segments[segments.length - 1].seq;
const lastRaw = await this.backend.read(validSegments[validSegments.length - 1].key); this.currentSegment = lastSeq;
const lastRaw = await this.backend.read(this.segmentKey(lastSeq));
this.currentSize = lastRaw ? lastRaw.byteLength : 0; this.currentSize = lastRaw ? lastRaw.byteLength : 0;
// 旧格式键存在时(迁移中),下一条记录另起分片,避免与旧键序号冲突 // 旧格式键存在时(迁移中),下一条记录另起分片,避免与旧键序号冲突
if (legacyKeys.length > 0) { if (legacyKeys.length > 0) {
@@ -133,13 +204,28 @@ export class SegmentedWALStore implements WALStore {
// 仅有旧格式:迁移中,新写入从分片 0 开始(checkpoint 会清空旧键) // 仅有旧格式:迁移中,新写入从分片 0 开始(checkpoint 会清空旧键)
this.currentSegment = 0; this.currentSegment = 0;
this.currentSize = 0; this.currentSize = 0;
} else if (fromSegment > 0) {
// 活跃区间的分片已被清理(例如上一次 truncateBefore 的删除生效):
// 新写入从当前分片号继续,避免复用已删除的历史分片号
this.currentSegment = fromSegment;
this.currentSize = 0;
} }
const total = parts.reduce((s, c) => s + c.byteLength, 0); const total = parts.reduce((s, c) => s + c.byteLength, 0);
const combined = new Uint8Array(total); const combined = new Uint8Array(total);
let off = 0; let off = 0;
for (const c of parts) { combined.set(c, off); off += c.byteLength; } for (const c of parts) { combined.set(c, off); off += c.byteLength; }
return combined; return {
data: combined,
fromSegment,
segments: readSegments,
gaps,
missingTail: 0,
};
}
async readAll(): Promise<Uint8Array> {
return (await this.readAllFrom(0)).data;
} }
async truncate(): Promise<void> { async truncate(): Promise<void> {
@@ -149,8 +235,142 @@ export class SegmentedWALStore implements WALStore {
if (walKeys.length > 0) { if (walKeys.length > 0) {
await this.backend.deleteMany(walKeys); await this.backend.deleteMany(walKeys);
} }
// v0.8.0:分片号只增不减(同 truncateBefore 的说明)—— 复用序号会让
// "同一序号对应两代记录",恢复按序号排序时旧记录可能排在新记录之后被重放。
const maxSeq = (await this.listSegments()).reduce((max, s) => Math.max(max, s.seq), -1);
this.currentSegment = Math.max(this.currentSegment, maxSeq + 1);
this.currentSize = 0;
this.segmentFirstLsn.clear();
}
/**
* v0.8.0:**整库清空**后重置分片编号(只有 `clearAll` 这种"介质被整体抹掉、
* manifest 也重新从 0 开始"的场景才允许调用)。
*/
reset(): void {
this.currentSegment = 0; this.currentSegment = 0;
this.currentSize = 0; this.currentSize = 0;
this.segmentFirstLsn.clear();
}
/**
* v0.8.0:删除**整体 LSN 都 <= durableLsn** 的分片(前缀删除)。
*
* 语义约束(调用方必须保证):`durableLsn` 之前的记录已存在于已提交的
* manifest/SSTable 中。返回仍需保留的最小分片号 —— 调用方应把它写进
* manifest 的 `wal.startSegment`,这样即便删除只完成了一半(崩溃/介质错误),
* 恢复也会忽略那些残留的旧世代分片。
*
* @param durableLsn 落盘水位(lsn <= 它的记录已确认存在于 SSTable 中)
* @param latestLsn 当前 LSN 高水位(调用方 flush 之后的 `wal.getLsn()`):
* 用于判断**最后一个分片**是否也已被水位完整覆盖。
* WAL 记录里只有"每条记录的 LSN",分片的**末条** LSN 无法
* 从分片首字节推出,而"最后一个分片之后没有分片"这一点只有
* 调用方知道 —— 缺了它就会出现"分片永远删不掉"(实测:
* close 之后 `__wal_000000.bin` 仍在,每次打开都要重读一遍)。
*/
/**
* v0.8.0**只计算**"还需要保留的最小分片号"(无副作用)。
*
* 为什么需要"先算后删":调用方必须**先把这个值提交进 manifest**,再删除分片。
* 顺序反了(先删后记录)会留下一个窗口:崩溃后 manifest 里的 `startSegment`
* 比介质上实际存在的分片更小,恢复时看到"前缀缺失"就无法区分
* "正常清理过的前缀"与"介质丢了一段记录"—— 前者无害,后者是数据丢失。
*/
async planKeepFrom(durableLsn: number, latestLsn?: number): Promise<number> {
const segments = await this.listSegments();
if (segments.length === 0) return 0;
// 判定每个分片是否"整段已被水位覆盖":
// - 非末分片:下一分片的首条记录 LSN <= durableLsn ⇒ 本分片全部记录都 <= 水位
// - 末分片:已知当前 LSN 高水位 <= durableLsn ⇒ 本分片全部记录都 <= 水位
for (let i = 0; i < segments.length; i++) {
const seg = segments[i];
const isLast = i === segments.length - 1;
const firstLsn = this.segmentFirstLsn.get(seg.seq);
// 未知边界:保守地从这里开始保留
if (firstLsn === undefined) return seg.seq;
const fullyCovered = isLast
? (typeof latestLsn === 'number' && latestLsn > 0 && latestLsn <= durableLsn)
: (() => {
const nextFirst = this.segmentFirstLsn.get(segments[i + 1].seq);
return nextFirst !== undefined && nextFirst <= durableLsn;
})();
if (!fullyCovered) return seg.seq;
}
// 全部分片都被水位覆盖
return segments[segments.length - 1].seq + 1;
}
async truncateBefore(durableLsn: number, latestLsn?: number): Promise<number> {
// v0.8.0:旧格式(__wal_N 每条一个 key)同样按水位清理 ——
// 它们是"迁移前"的记录,与分片一样只在 lsn <= durableLsn 时才可删。
// 修复前这些键只在整个 WAL 被 truncate() 时才清,导致已落盘的旧记录
// 永远留在介质上(每次打开都会被 readAll 读出来再按 lsn 跳过)。
//
// 注意:必须在"没有新格式分片"的早退**之前**处理 —— 只含旧格式的库
//(v0.4.4 升级现场)恰恰是这条路径最常见的输入。
const legacyKeys = (await this.backend.listKeys()).filter((k) => LEGACY_RECORD_REGEX.test(k));
if (legacyKeys.length > 0) {
const obsoleteLegacy: string[] = [];
for (const key of legacyKeys) {
const raw = await this.backend.read(key);
if (!raw || raw.byteLength < 4) {
obsoleteLegacy.push(key); // 空/残缺:无记录可保留
continue;
}
const firstLsn = new DataView(raw, 0, 4).getUint32(0, false);
if (firstLsn <= durableLsn) obsoleteLegacy.push(key);
}
if (obsoleteLegacy.length > 0) {
try {
await this.backend.deleteMany([...obsoleteLegacy, LEGACY_COUNT_KEY]);
} catch { /* 同上:残留无害(恢复按 lsn 跳过) */ }
}
}
const segments = await this.listSegments();
if (segments.length === 0) {
this.currentSegment = 0;
this.currentSize = 0;
return 0;
}
const keepFrom = await this.planKeepFrom(durableLsn, latestLsn);
const obsolete = segments.filter((s) => s.seq < keepFrom);
if (obsolete.length > 0) {
try {
await this.backend.deleteMany(obsolete.map((s) => s.key));
} catch { /* 删除失败:残留分片由 manifest.startSegment 在恢复时忽略 */ }
for (const s of obsolete) this.segmentFirstLsn.delete(s.seq);
}
// 全部删除(含当前分片)→ **分片号继续往后走,绝不重置回 0**。
//
// 这是 v0.8.0 的一处关键修正:修复前这里是 `currentSegment = 0`。而 manifest
// 里的 `startSegment` 是**删除前**就算好并提交的(= 最大分片号 + 1),于是
// 重置之后新记录写进分片 0,恢复时从 startSegment 开始读 → 整批新记录被跳过
// (实测随机压力用例:删掉的行复活 / 已确认写入丢失,两种方向都出现过)。
// 更糟的是"旧世代排在最新记录之后":分片号复用会让同一序号对应两代记录,
// 恢复按序号排序时旧 INSERT 会排在一次 DELETE 之后被重放。
//
// 现在分片号只增不减:删除只回收空间,不复用标识。
if (obsolete.length === segments.length) {
const maxSeq = segments[segments.length - 1].seq;
const leftover = await this.listSegments();
if (leftover.length === 0) {
this.currentSegment = Math.max(maxSeq + 1, keepFrom);
this.currentSize = 0;
this.segmentFirstLsn.clear();
return keepFrom;
}
// 删除失败留下残片:它们都在 startSegment 之前(恢复时被忽略),
// 新记录必须继续用**更大的**分片号,避免与残片混进同一个文件。
this.currentSegment = Math.max(leftover[leftover.length - 1].seq + 1, keepFrom);
this.currentSize = 0;
return keepFrom;
}
return keepFrom;
} }
async exists(): Promise<boolean> { async exists(): Promise<boolean> {
+8 -5
View File
@@ -9,7 +9,7 @@
import { AriaEngine } from '../../src/engine/aria/index'; import { AriaEngine } from '../../src/engine/aria/index';
import { createSchema } from '../../src/table/schema'; import { createSchema } from '../../src/table/schema';
import { resetOPFSMock } from '../helpers/storage-harness'; import { resetOPFSMock, readManifestNamespace } from '../helpers/storage-harness';
beforeEach(() => { resetOPFSMock(); }); beforeEach(() => { resetOPFSMock(); });
@@ -41,12 +41,15 @@ async function listSSTKeys(engine: AriaEngine): Promise<string[]> {
return keys.filter((k) => k.startsWith('pg_')); return keys.filter((k) => k.startsWith('pg_'));
} }
/** 读取主 LSM 的 SSTable meta 列表 */ /**
* 读取主 LSM 的 SSTable meta 列表(v0.8.0 B-6:改读 manifest)。
*
* 元数据不再是独立的裸 JSON:它随 manifest 原子提交(带 CRC + 世代号)。
* 旧布局下 `JSON.parse` 失败会被当成"没有文件",即元数据损坏 = 静默空库。
*/
async function listSSTMetas(engine: AriaEngine): Promise<{ id: number; pageIds?: number[] }[]> { async function listSSTMetas(engine: AriaEngine): Promise<{ id: number; pageIds?: number[] }[]> {
const backend = (engine as any).backend; const backend = (engine as any).backend;
const raw = await backend.read('__aria_lsm_meta'); return readManifestNamespace(backend, 'main');
if (!raw) return [];
return JSON.parse(new TextDecoder().decode(raw)) as { id: number; pageIds?: number[] }[];
} }
describe('AriaEngine — SSTable CRC 损坏检测(集成)', () => { describe('AriaEngine — SSTable CRC 损坏检测(集成)', () => {
+11 -1
View File
@@ -58,7 +58,17 @@ async function writeAndMeasure(opts: {
await engine.createTable(SCHEMA() as never); await engine.createTable(SCHEMA() as never);
for (let i = 0; i < rows; i++) await engine.insert('t', [{ id: `k${i}`, blob: COMPRESSIBLE }]); for (let i = 0; i < rows; i++) await engine.insert('t', [{ id: `k${i}`, blob: COMPRESSIBLE }]);
const lsm = (engine as unknown as { lsm: { sstableStore: { listMeta(): Promise<Array<{ totalSize: number }>> } } }).lsm; const lsm = (engine as unknown as {
lsm: { flush(): Promise<void>; sstableStore: { listMeta(): Promise<Array<{ totalSize: number }>> } };
}).lsm;
// v0.8.0B-6):**测量前显式 flush**。
//
// 此前这里直接读 meta 求和,于是"落盘字节数"取决于测量瞬间有多少数据恰好在
// SSTable 里 —— 而 manifest 提交(单一提交点)让每次 flush 多一次原子提交,
// 后台 flush 的进度随之变化,两次实验的"已落盘比例"不再相同,比值就变成在
// 测时序而不是测压缩(实测:未 flush 时 off=19040/on=5130flush 后
// off=58570/on=10096 —— 后者才是同一份数据的真实压缩率)。
await lsm.flush();
const metas = await lsm.sstableStore.listMeta(); const metas = await lsm.sstableStore.listMeta();
const storedBytes = metas.reduce((sum, m) => sum + (m.totalSize ?? 0), 0); const storedBytes = metas.reduce((sum, m) => sum + (m.totalSize ?? 0), 0);
+9 -8
View File
@@ -9,7 +9,7 @@ import { AriaEngine } from '../../src/engine/aria/index';
import { createSchema } from '../../src/table/schema'; import { createSchema } from '../../src/table/schema';
import { MetonaSqlark } from '../../src/core'; import { MetonaSqlark } from '../../src/core';
import { resetOPFSMock } from '../helpers/storage-harness'; import { resetOPFSMock, readManifestState } from '../helpers/storage-harness';
beforeEach(() => { resetOPFSMock(); }); beforeEach(() => { resetOPFSMock(); });
@@ -455,13 +455,14 @@ describe('AriaEngine — Schema 持久化 (Memory Backend)', () => {
name: { type: 'string', required: true }, name: { type: 'string', required: true },
})); }));
// 直接通过 backend 验证 Schema JSON 已写入 // v0.8.0(B-6):表结构不再是独立的裸 JSON__aria_schemas),
const raw = await (engine as any).backend.read('__aria_schemas'); // 而是随 manifest 一起**原子提交**(单一提交点)。测试改读 manifest ——
expect(raw).not.toBeNull(); // 它才是落盘结构的权威来源(带 CRC 与世代号)。
const json = new TextDecoder().decode(raw); // 旧布局的问题:坏 JSON 会被 `readMetaList()` 当成 `[]`,元数据损坏 = 静默空库。
const data = JSON.parse(json); const manifest = await readManifestState((engine as any).backend);
expect(data.users).toBeDefined(); expect(manifest).not.toBeNull();
expect(data.users.id.primaryKey).toBe(true); expect(manifest!.schemas.users).toBeDefined();
expect(manifest!.schemas.users.id.primaryKey).toBe(true);
await engine.close(); await engine.close();
}); });
+33
View File
@@ -518,3 +518,36 @@ export class CrashableStoreBackend implements IStorageBackend {
this.store.commitAll(); this.store.commitAll();
} }
} }
// ---------------------------------------------------------------------------
// v0.8.0B-6):manifest 读取辅助
// ---------------------------------------------------------------------------
/**
* 读取存储上**最新有效世代**的 manifest(用生产解码器,不做测试特供路径)。
*
* 为什么测试需要它:v0.8.0 起 SSTable 元数据与表结构不再是各自独立的裸 JSON
* `__aria_lsm_meta*` / `__aria_schemas`),而是随 manifest 一起原子提交。
* 任何"直接读裸 JSON 验证落盘内容"的测试都必须改读 manifest —— 否则它验证的是
* 一个已经不存在(且没有 CRC/世代保护)的存储布局。
*
* @returns manifest 内容;全新库(没有任何 manifest 文件)返回 null
* 文件存在但全部世代无效 → 抛错(与生产恢复语义一致,绝不"失败当空库")
*/
export async function readManifestState(
backend: IStorageBackend,
): Promise<import('../../src/engine/aria/store/manifest').AriaManifest | null> {
const { ManifestStore } = await import('../../src/engine/aria/store/manifest');
const store = new ManifestStore({ backend });
const loaded = await store.load();
return loaded.manifest;
}
/** 读取某个命名空间当前的 SSTable meta 列表(测试断言用) */
export async function readManifestNamespace(
backend: IStorageBackend,
ns: string = 'main',
): Promise<{ id: number; level: number; pageIds?: number[]; minKey: string; maxKey: string }[]> {
const manifest = await readManifestState(backend);
return manifest?.namespaces[ns]?.sstables ?? [];
}
+5 -7
View File
@@ -20,7 +20,7 @@ import { OPFSBackend } from '../src/engine/aria/store/opfs_backend';
import { KVStoreEngine } from '../src/engine/kvstore_engine'; import { KVStoreEngine } from '../src/engine/kvstore_engine';
import type { SSTableMeta } from '../src/engine/aria/types'; import type { SSTableMeta } from '../src/engine/aria/types';
import { resetOPFSMock } from './helpers/storage-harness'; import { resetOPFSMock, readManifestNamespace } from './helpers/storage-harness';
beforeEach(() => { resetOPFSMock(); }); beforeEach(() => { resetOPFSMock(); });
@@ -153,8 +153,8 @@ describe('P0-1b — AriaEngine 打开时完整性校验', () => {
// 篡改存储:把第一个 SSTable 的第一个页面写成残缺内容(meta 仍引用它) // 篡改存储:把第一个 SSTable 的第一个页面写成残缺内容(meta 仍引用它)
const backend = new OPFSBackend(); const backend = new OPFSBackend();
await backend.open(dbName); await backend.open(dbName);
const metas = JSON.parse(new TextDecoder().decode( // v0.8.0B-6):SSTable meta 随 manifest 原子提交(不再是裸 JSON key)
await backend.read('__aria_lsm_meta') as ArrayBuffer)) as { id: number; pageIds: number[] }[]; const metas = await readManifestNamespace(backend, 'main') as { id: number; pageIds: number[] }[];
expect(metas.length).toBeGreaterThan(0); expect(metas.length).toBeGreaterThan(0);
const pageId = metas[0].pageIds[0]; const pageId = metas[0].pageIds[0];
await backend.write(`pg_${pageId}`, new TextEncoder().encode('truncated-garbage').buffer); await backend.write(`pg_${pageId}`, new TextEncoder().encode('truncated-garbage').buffer);
@@ -174,8 +174,7 @@ describe('P0-1b — AriaEngine 打开时完整性校验', () => {
const backend2 = new OPFSBackend(); const backend2 = new OPFSBackend();
await backend2.open(dbName); await backend2.open(dbName);
const remaining = (await backend2.listKeys()).filter((k) => k.startsWith('pg_')); const remaining = (await backend2.listKeys()).filter((k) => k.startsWith('pg_'));
const metaRaw = await backend2.read('__aria_lsm_meta'); const metaList = await readManifestNamespace(backend2, 'main');
const metaList = JSON.parse(new TextDecoder().decode(metaRaw ?? new Uint8Array())) as { pageIds?: number[] }[];
const livePages = new Set<number>(); const livePages = new Set<number>();
for (const m of metaList) if (m.pageIds) for (const pid of m.pageIds) livePages.add(pid); for (const m of metaList) if (m.pageIds) for (const pid of m.pageIds) livePages.add(pid);
const orphan = remaining.filter((k) => !livePages.has(Number(k.slice(3)))); const orphan = remaining.filter((k) => !livePages.has(Number(k.slice(3))));
@@ -414,8 +413,7 @@ describe('P2-9 — 统一自愈接口 repair / clearAll', () => {
// 篡改一个 SSTable 的页面文件 // 篡改一个 SSTable 的页面文件
const backend = new OPFSBackend(); const backend = new OPFSBackend();
await backend.open(dbName); await backend.open(dbName);
const metas = JSON.parse(new TextDecoder().decode( const metas = await readManifestNamespace(backend, 'main') as { id: number; pageIds: number[] }[];
await backend.read('__aria_lsm_meta') as ArrayBuffer)) as { id: number; pageIds: number[] }[];
expect(metas.length).toBeGreaterThan(0); expect(metas.length).toBeGreaterThan(0);
const victimPage = metas[0].pageIds[0]; const victimPage = metas[0].pageIds[0];
const raw = new Uint8Array(await backend.read(`pg_${victimPage}`) as ArrayBuffer); const raw = new Uint8Array(await backend.read(`pg_${victimPage}`) as ArrayBuffer);
+3 -3
View File
@@ -116,7 +116,8 @@ describe('P1-A — 二级索引恢复', () => {
const idxLsm = (engine2 as any).secondaryIndexes.get('users:idx:email'); const idxLsm = (engine2 as any).secondaryIndexes.get('users:idx:email');
expect(idxLsm).toBeDefined(); expect(idxLsm).toBeDefined();
expect(idxLsm.getStats().sstableCount).toBeGreaterThan(0); expect(idxLsm.getStats().sstableCount).toBeGreaterThan(0);
await idxLsm.prefetchRange('', '\uffff'); // v0.8.0B-6/55):`prefetchRange` 已删除 —— 读取自洽(未命中即回源 + CRC
// 校验),调用方不再需要"先预加载再读"这条隐式约定。
// v0.8.0: LSM.rangeScan 改为 async(读取自洽,未命中会回源加载) // v0.8.0: LSM.rangeScan 改为 async(读取自洽,未命中会回源加载)
expect(await idxLsm.rangeScan('', '\uffff')).toHaveLength(3); expect(await idxLsm.rangeScan('', '\uffff')).toHaveLength(3);
const byEmail = await engine2.find('users', { table: 'users', where: { email: 'a@x.com' } }); const byEmail = await engine2.find('users', { table: 'users', where: { email: 'a@x.com' } });
@@ -147,8 +148,7 @@ describe('P1-A — 二级索引恢复', () => {
// 强断言:索引 LSM 已恢复且包含 WAL 回放的行(崩溃前索引未更新,恢复后必须重建) // 强断言:索引 LSM 已恢复且包含 WAL 回放的行(崩溃前索引未更新,恢复后必须重建)
const idxLsm = (engine2 as any).secondaryIndexes.get('users:idx:city'); const idxLsm = (engine2 as any).secondaryIndexes.get('users:idx:city');
expect(idxLsm).toBeDefined(); expect(idxLsm).toBeDefined();
await idxLsm.prefetchRange('', '\uffff'); // v0.8.0B-6/55):prefetchRange 已删除(读取自洽);rangeScan 现为 async
// v0.8.0: LSM.rangeScan 改为 async
expect(await idxLsm.rangeScan('', '\uffff')).toHaveLength(2); expect(await idxLsm.rangeScan('', '\uffff')).toHaveLength(2);
// 索引查询应看到 WAL 恢复的行(修复前索引与主数据不一致 → 丢行) // 索引查询应看到 WAL 恢复的行(修复前索引与主数据不一致 → 丢行)
const byCity = await engine2.find('users', { table: 'users', where: { city: 'Shanghai' } }); const byCity = await engine2.find('users', { table: 'users', where: { city: 'Shanghai' } });
+14 -2
View File
@@ -120,8 +120,20 @@ describe('P0 — flush 报告后台失败', () => {
await new Promise((r) => setTimeout(r, 50)); await new Promise((r) => setTimeout(r, 50));
// flush 必须报告后台失败(修复前静默吞错) // flush 必须报告后台失败(修复前静默吞错)
await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' }); await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' });
// 再次 flush:错误已消费,正常完成
await expect(lsm.flush()).resolves.toBeUndefined(); // v0.8.0B-6/44+45)契约变更:第二次 flush **同样必须失败**。
//
// 修复前的第二行断言是 `resolves.toBeUndefined()` —— 它编码的语义是
// "错误已消费 → 这次 flush 算成功"。但 `FailingStore.save` 是**永远**失败,
// 那份数据此时仍然只在内存里(WAL 之外没有任何副本):报告"成功"等于告诉
// 调用方"已落盘",而崩溃就会丢。这属于"静默成功",与本次根治的目标正好相反。
//
// 现在的语义:flush 只有在**真的把数据落盘**后才 resolve;失败可以重试
// (冻结表会被重新入链,见下一条用例),但重试仍失败就必须继续报错。
await expect(lsm.flush()).rejects.toMatchObject({ code: 'ARIA_BACKGROUND_ERROR' });
// 冻结表仍在(可读、可重试),数据没有凭空消失
expect(lsm.getStats().frozenTables).toBeGreaterThan(0);
expect(await lsm.get('k0')).toEqual({ v: 0 });
}); });
it('后台 compaction 失败后 flush() 报告(链不卡死)', async () => { it('后台 compaction 失败后 flush() 报告(链不卡死)', async () => {
+72 -6
View File
@@ -29,6 +29,13 @@ import { describe, it, expect, beforeEach } from '@jest/globals';
import { AriaEngine } from '../src/engine/aria/index'; import { AriaEngine } from '../src/engine/aria/index';
import { resetOPFSMock } from './helpers/storage-harness'; import { resetOPFSMock } from './helpers/storage-harness';
import type { IStorageBackend } from '../src/engine/aria/store/backend'; import type { IStorageBackend } from '../src/engine/aria/store/backend';
import {
decodeManifest,
encodeManifest,
generationFromKey,
manifestKey,
type AriaManifest,
} from '../src/engine/aria/store/manifest';
beforeEach(() => { resetOPFSMock(); }); beforeEach(() => { resetOPFSMock(); });
@@ -71,16 +78,71 @@ class OrderRecordingBackend implements IStorageBackend {
async exists(k: string): Promise<boolean> { return this.files.has(k); } async exists(k: string): Promise<boolean> { return this.files.has(k); }
async clear(): Promise<void> { this.files.clear(); } async clear(): Promise<void> { this.files.clear(); }
/** 索引:WAL 记录写入发生在 schema 落盘**之前** */ /**
* WAL schema ****
*
* v0.8.0B-6schema `__aria_schemas` JSON
* manifest "schema 落盘" `__aria_manifest_*`
* ****WAL
*/
walPrecedesSchema(): boolean { walPrecedesSchema(): boolean {
const wal = this.order.findIndex((k) => k.includes('wal')); const wal = this.order.findIndex((k) => k.includes('wal'));
const sch = this.order.findIndex((k) => k.includes('schemas')); const sch = this.order.findIndex((k) => k.includes('__aria_manifest_'));
return wal >= 0 && sch >= 0 && wal < sch; return wal >= 0 && sch >= 0 && wal < sch;
} }
/** 模拟"该文件没能落盘"(崩溃窗口) */ /** 模拟"该文件没能落盘"(崩溃窗口) */
dropFile(pattern: string): void { dropFile(pattern: string): void {
for (const k of [...this.files.keys()]) if (k.includes(pattern)) this.files.delete(k); for (const k of [...this.files.keys()]) if (k.includes(pattern)) this.files.delete(k);
} }
/**
* manifest **** +
* "DDL 之前" `rollbackManifest()`
*/
snapshotManifest(): { manifest: AriaManifest; generation: number } {
const gens = [...this.files.keys()]
.map((k) => generationFromKey(k))
.filter((g): g is number => g !== null)
.sort((a, b) => b - a);
if (gens.length === 0) throw new Error('no manifest on medium');
const raw = this.files.get(manifestKey(gens[0]))!;
const decoded = decodeManifest(new Uint8Array(raw));
if (!decoded.ok) throw new Error(`manifest decode failed: ${decoded.reason}`);
return { manifest: decoded.manifest, generation: gens[0] };
}
/**
* manifest ****
*
* "这次 DDL 的 manifest 提交从未落盘" WAL
* **WAL **
*
* 2
*
*/
rollbackManifest(snapshot: { manifest: AriaManifest; generation: number }): void {
const gens = [...this.files.keys()]
.map((k) => generationFromKey(k))
.filter((g): g is number => g !== null);
const nextGeneration = Math.max(0, ...gens) + 1;
const rolled: AriaManifest = {
...snapshot.manifest,
generation: nextGeneration,
namespaces: snapshot.manifest.namespaces,
schemas: snapshot.manifest.schemas,
wal: { ...snapshot.manifest.wal },
frozen: [],
owner: { ...snapshot.manifest.owner },
};
const bytes = encodeManifest(rolled);
this.files.set(
manifestKey(nextGeneration),
bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer,
);
for (const k of [...this.files.keys()]) {
const gen = generationFromKey(k);
if (gen !== null && gen !== nextGeneration) this.files.delete(k);
}
}
} }
/** /**
@@ -148,13 +210,15 @@ describe('[v0.8.0] A41 DDL 结构变更可崩溃恢复(WAL 兜底)', () => {
const backend = new OrderRecordingBackend(); const backend = new OrderRecordingBackend();
const engine = openWith(backend); const engine = openWith(backend);
await engine.open('ddl-alter-crash', 1); await engine.open('ddl-alter-crash', 1);
// 记录"DDL 之前"的 manifest 状态:崩溃窗口 = 之后的提交都没落盘
const beforeDdl = backend.snapshotManifest();
await engine.createTable(schema('t')); await engine.createTable(schema('t'));
await engine.alterTable('t', 'ADD', { name: 'tag', type: 'string' } as never); await engine.alterTable('t', 'ADD', { name: 'tag', type: 'string' } as never);
await engine.alterTable('t', 'ADD', { name: 'tag2', type: 'string' } as never); await engine.alterTable('t', 'ADD', { name: 'tag2', type: 'string' } as never);
// 崩溃窗口:两次 ALTER 的 schema 都没能落盘(不 close,否则 close 会重试落盘) // 崩溃窗口:两次 ALTER 的 schema 都没能落盘(不 close,否则 close 会重试落盘)
backend.dropFile('__aria_schemas'); backend.rollbackManifest(beforeDdl);
expect((await backend.listKeys()).some((k) => k.includes('schemas'))).toBe(false); expect(backend.snapshotManifest().generation).toBeGreaterThan(beforeDdl.generation);
const engine2 = openWith(backend); const engine2 = openWith(backend);
await engine2.open('ddl-alter-crash', 1); await engine2.open('ddl-alter-crash', 1);
@@ -173,9 +237,10 @@ describe('[v0.8.0] A41 DDL 结构变更可崩溃恢复(WAL 兜底)', () => {
await engine.insert('t', [{ id: 'r1' }]); await engine.insert('t', [{ id: 'r1' }]);
await engine.createTable(schema('other')); await engine.createTable(schema('other'));
await engine.insert('other', [{ id: 'o1' }]); await engine.insert('other', [{ id: 'o1' }]);
const beforeDrop = backend.snapshotManifest();
await engine.dropTable('other'); await engine.dropTable('other');
backend.dropFile('__aria_schemas'); // schema 落盘丢失 backend.rollbackManifest(beforeDrop); // DROP 的持久化提交丢失
// 不 close(崩溃语义) // 不 close(崩溃语义)
const engine2 = openWith(backend); const engine2 = openWith(backend);
@@ -191,9 +256,10 @@ describe('[v0.8.0] A41 DDL 结构变更可崩溃恢复(WAL 兜底)', () => {
const backend = new OrderRecordingBackend(); const backend = new OrderRecordingBackend();
const engine = openWith(backend); const engine = openWith(backend);
await engine.open('ddl-create-crash', 1); await engine.open('ddl-create-crash', 1);
const beforeCreate = backend.snapshotManifest();
await engine.createTable(schema('t')); await engine.createTable(schema('t'));
await engine.insert('t', [{ id: 'r1' }]); await engine.insert('t', [{ id: 'r1' }]);
backend.dropFile('__aria_schemas'); backend.rollbackManifest(beforeCreate);
const engine2 = openWith(backend); const engine2 = openWith(backend);
await engine2.open('ddl-create-crash', 1); await engine2.open('ddl-create-crash', 1);
File diff suppressed because it is too large Load Diff