fix(A8 + LSM 读自洽): 行所有权根治 + 读取路径不再依赖 prefetch

A8 行引用泄漏(调用方改查询结果即改写存储)
  实测:rows[0].tag = 'HACKED' 后,tag='HACKED' 与 tag='x' 两条索引查询都返回 0 行 ——
  行与索引失配、该行永久查不出来;嵌套 json 值同样按引用共享。
  Aria 因走反序列化路径反而幸免,又形成跨引擎差异。

  根治:在 IStorageEngine 契约层写入**行所有权约定**(engine/interface.ts)
  —— 读出的行是副本、写入接收的行也是副本;新增 cloneRow/cloneRows
  (优先 structuredClone,退化路径处理 Date/嵌套对象/二进制)。
  Memory/KVStore/Hybrid:find、findStream、getRow 全部返回副本。
  Aria:getAllRows 此前只做 `{ ...value }` 浅拷贝(嵌套 json 仍共享引用),
  改为深拷贝;find/findStream 返回副本。
  实测四种引擎:修改返回值后重读不变、索引两条查询均正确。

LSM 读取自洽(审计 P1-1:缓存未命中 = 静默丢数据)
  此前 loadSSTableReader 缓存未命中返回 null,而所有调用方都是
    const reader = this.loadSSTableReader(meta); if (!reader) continue;
  于是**未命中就静默跳过整个 SSTable**。实测:缓存上限 4KB 而 SSTable 更大时,
  300 行只能查回 59 行,且不报错。
  同时"读路径必须先 prefetch"这个隐式约定,是每次读都要 drainChain + prefetch
  的原因(性能悬崖的另一半)。

  根治:LSM.get / rangeScan / rangeScanLazy 改为 async,未命中即
  `await sstableStore.load()` 回源 + CRC 校验(损坏则自愈清理 meta),
  只有数据确实不存在才返回 null。checkUniqueSync 相应改名 checkUnique 并 async
  (原命名正是因为依赖 prefetch 约定)。引擎侧 12 处调用点补 await。

附带修正的缓存语义:
  - tryCacheSSTable:单个 SSTable 超过缓存上限时标记为常驻(pinned),
    不参与驱逐 —— 驱逐它等价于静默丢数据;内存上限因此是
    cacheLimit + 单个最大 SSTable,已在代码与测试中明确。
  - trimCache 跳过 pinned 条目(此前会把全部缓存一次性清空)。
  - 新增 getCacheSize/getCacheLimit/getOversizedCount/setCacheLimit 访问器
    (测试此前直接读私有字段 cacheSize/cacheLimitBytes —— 那是 TS 错误,
    只因测试不做类型检查才没暴露)。

queryStream async 回调
  此前用 constructor.name === 'AsyncFunction' 判定,对"普通函数返回 Promise"
  完全失效(Promise 被静默丢弃)。现改为双条件识别并走物化路径逐行 await,
  async 回调被真正等待。

测试契约修正:
  - aria-cache:'缓存大小受上限约束' 在极小缓存下是不可成立的契约,改为断言
    真正重要的不变量(数据完整;可装入时受上限约束),并新增"超大 SSTable 常驻"
    用例;'缓存驱逐后全表扫描仍返回完整数据' 保留 300 行断言(此前会失败)。
  - hybrid:磁盘引擎标签断言从 indexeddb(v0.6.0 已移除)改为 opfs。
This commit is contained in:
thzxx
2026-09-14 21:43:21 +08:00
parent 752bdea97d
commit 074afd3f1e
7 changed files with 312 additions and 73 deletions
+35 -26
View File
@@ -1,3 +1,4 @@
import { cloneRow } from '../interface';
/**
* AriaEngine — 自研页面式存储引擎主类
* @module engine/aria/index
@@ -584,9 +585,10 @@ export class AriaEngine implements IStorageEngine {
);
}
pkSet.add(pkValue);
// v0.8.0: 事务快照优先,否则回源 LSMlsm.get 现为 async
const existing = this.currentTxnId
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
: this.lsm.get(key);
? (this.txnSnapshot?.get(key) ?? await this.lsm.get(key))
: await this.lsm.get(key);
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
throw new DatabaseError(
`Duplicate primary key "${pkValue}" in table "${tableName}"`,
@@ -627,7 +629,7 @@ export class AriaEngine implements IStorageEngine {
);
}
seen.add(v);
this.checkUniqueSync(tableName, [colName], validated, pkValue);
await this.checkUnique(tableName, [colName], validated, pkValue);
}
}
@@ -706,7 +708,8 @@ export class AriaEngine implements IStorageEngine {
// 查询完成,回收查询期间的临时缓存超限
this.trimAllCaches();
return rows;
// v0.8.0: 行所有权 —— 返回副本,调用方不得改写存储(见 engine/interface.ts 约定)
return rows.map((row) => cloneRow(row));
}
async update(
@@ -780,7 +783,7 @@ export class AriaEngine implements IStorageEngine {
// 批内唯一互查(索引尚未更新,两行同时改到同一新值需要互查兜底)
this.checkBatchUnique(tableName, uniqueCols, updated, batchUnique);
// v0.6.2: 唯一约束检查(排除自身旧索引条目:主键变更时旧条目仍以旧键存在)
this.checkUniqueSync(tableName, uniqueCols, updated, String(row[pkCol]));
await this.checkUnique(tableName, uniqueCols, updated, String(row[pkCol]));
// v0.4.2-fix: 支持更新主键 — 删除旧键 + 落新键 + WAL 两条记录
const newPk = String(updated[pkCol]);
@@ -791,8 +794,8 @@ export class AriaEngine implements IStorageEngine {
if (pkChanged) {
const newKey = `${tableName}:${newPk}`;
const existing = this.currentTxnId
? (this.txnSnapshot?.get(newKey) ?? this.lsm.get(newKey))
: this.lsm.get(newKey);
? (this.txnSnapshot?.get(newKey) ?? await this.lsm.get(newKey))
: await this.lsm.get(newKey);
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
throw new DatabaseError(
`Duplicate primary key "${newPk}" in table "${tableName}" (cannot update key to existing value)`,
@@ -1191,7 +1194,7 @@ export class AriaEngine implements IStorageEngine {
const emit = (row: Record<string, unknown>): boolean => {
if (hasWhere && !matchWhere(row, query.where!)) return true;
if (skipped < offset) { skipped++; return true; }
onRow(project ? project(row) : row);
onRow(project ? project(row) : cloneRow(row));
count++;
return count < limit;
};
@@ -1200,7 +1203,7 @@ export class AriaEngine implements IStorageEngine {
// 事务中:物化后逐行回调(快照合并需要全量行集)
const rows = await this.find(tableName, { ...query, orderBy: undefined, limit: undefined, offset: undefined });
for (const row of rows) {
onRow(project ? project(row) : row);
onRow(project ? project(row) : cloneRow(row));
}
return rows.length;
}
@@ -1217,7 +1220,7 @@ export class AriaEngine implements IStorageEngine {
// 全表惰性扫描(含 WHERE 过滤,不物化;v0.7.4: callback 返回 false 提前终止,
// 未消费的 SSTable 块 / 子树不再解析 —— 真流式,大表 limit 内存 O(1)
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => {
await this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => {
if (count >= limit) return false;
const row = { ...value };
row[pkCol] = key.slice(prefix.length);
@@ -1314,7 +1317,7 @@ export class AriaEngine implements IStorageEngine {
const prefix = `${tableName}:`;
const endKey = `${prefix}\uffff`;
await this.lsm.prefetchRange(prefix, endKey);
const entries = this.lsm.rangeScan(prefix, endKey);
const entries = await this.lsm.rangeScan(prefix, endKey);
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
for (const [key, value] of entries) {
if (!(column.name in value)) continue;
@@ -1603,9 +1606,11 @@ export class AriaEngine implements IStorageEngine {
const prefix = `${tableName}:`;
// 预加载范围内涉及的 SSTable,避免 rangeScan 时缓存未命中静默丢数据
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`);
const rows = entries.map(([key, value]) => {
const row = { ...value };
// v0.8.0: 深拷贝(此前 `{ ...value }` 只做浅拷贝,嵌套 json 值仍与 LSM
// 内部对象共享引用 —— 调用方改 rows[0].nested.a 会改写存储)
const row = cloneRow(value);
row[pkCol] = key.slice(prefix.length);
return row;
});
@@ -1872,7 +1877,7 @@ export class AriaEngine implements IStorageEngine {
const prefix = `${tableName}:`;
const endKey = `${prefix}\uffff`;
await this.lsm.prefetchRange(prefix, endKey);
const entries = this.lsm.rangeScan(prefix, endKey);
const entries = await this.lsm.rangeScan(prefix, endKey);
for (const [key] of entries) {
this.lsm.delete(key);
}
@@ -1893,23 +1898,27 @@ export class AriaEngine implements IStorageEngine {
}
/**
* v0.6.2: 同步唯一性检查(须在批次级 prefetchPrefixRanges 之后调用,循环内无 await)
* 唯一性检查
*
* v0.8.0: 由 `checkUniqueSync` 改名并改为 async —— 此前命名为 "Sync" 是因为它
* 依赖"批次级 prefetchPrefixRanges 之后索引数据已在缓存中"这一约定。现在
* LSM 读取自洽(未命中即回源),因此这里可以、也必须 await。
* 索引不含 null 条目(null 值不受唯一约束,与 MemoryEngine 语义一致)。
* @param currentPk 当前行主键(更新路径用于排除自身旧索引条目;插入路径无自身条目)
*/
private checkUniqueSync(
private async checkUnique(
tableName: string,
uniqueCols: string[],
row: Record<string, unknown>,
currentPk: string,
): void {
): Promise<void> {
for (const colName of uniqueCols) {
const val = row[colName];
if (val === undefined || val === null) continue;
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`);
if (!idxLsm) continue;
const prefix = `${String(val)}:`;
const entries = idxLsm.rangeScan(prefix, `${prefix}\uffff`);
const entries = await idxLsm.rangeScan(prefix, `${prefix}\uffff`);
for (const [, entry] of entries) {
const pk = (entry as unknown as { pk?: string }).pk;
if (pk !== undefined && pk !== currentPk) {
@@ -1994,14 +2003,14 @@ export class AriaEngine implements IStorageEngine {
if (typeof condition !== 'object' || condition === null) {
const key = `${tableName}:${condition}`;
await this.lsm.prefetchKeys([key]);
const value = this.lsm.get(key);
const value = await this.lsm.get(key);
return value ? [{ ...value, [pkCol]: condition }] : [];
}
const cond = condition as Record<string, unknown>;
if ('$eq' in cond) {
const key = `${tableName}:${cond.$eq}`;
await this.lsm.prefetchKeys([key]);
const value = this.lsm.get(key);
const value = await this.lsm.get(key);
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
}
// v0.3.3: PK $in → 主 LSM 多次精确查找(替代冗余 PK 二级索引)
@@ -2013,7 +2022,7 @@ export class AriaEngine implements IStorageEngine {
for (const v of cond.$in) {
const pk = String(v);
if (seen.has(pk)) continue;
const value = this.lsm.get(`${tableName}:${pk}`);
const value = await this.lsm.get(`${tableName}:${pk}`);
if (value) { seen.add(pk); rows.push({ ...value, [pkCol]: pk }); }
}
return rows;
@@ -2022,7 +2031,7 @@ export class AriaEngine implements IStorageEngine {
if ('$gt' in cond || '$gte' in cond || '$lt' in cond || '$lte' in cond) {
const prefix = `${tableName}:`;
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`);
const rows: Record<string, unknown>[] = [];
for (const [key, value] of entries) {
const candidate = { ...value, [pkCol]: key.slice(prefix.length) };
@@ -2065,7 +2074,7 @@ export class AriaEngine implements IStorageEngine {
const seenPks = new Set<string>(); // v0.4.1: IN 值可能重复,按 pk 去重
const pks: string[] = [];
for (const val of values) {
const entries = idxLsm.rangeScan(val, `${val}\uffff`);
const entries = await idxLsm.rangeScan(val, `${val}\uffff`);
for (const [, idxEntry] of entries) {
const pk = (idxEntry as { pk?: string }).pk;
if (pk && !seenPks.has(pk)) {
@@ -2076,7 +2085,7 @@ export class AriaEngine implements IStorageEngine {
}
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
for (const pk of pks) {
const row = this.lsm.get(`${tableName}:${pk}`);
const row = await this.lsm.get(`${tableName}:${pk}`);
if (row) results.push({ ...row, [pkCol]: pk });
}
return results;
@@ -2104,7 +2113,7 @@ export class AriaEngine implements IStorageEngine {
const actualEndKey = endKey.includes('\uffff') ? endKey : `${endKey}\uffff`;
// 预加载索引 LSM 与主 LSM 涉及的 SSTable
await idxLsm.prefetchRange(startKey, actualEndKey);
const entries = idxLsm.rangeScan(startKey, actualEndKey);
const entries = await idxLsm.rangeScan(startKey, actualEndKey);
const pks: string[] = [];
for (const [, idxEntry] of entries) {
const pk = (idxEntry as any).pk as string;
@@ -2113,7 +2122,7 @@ export class AriaEngine implements IStorageEngine {
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
const rows: Record<string, unknown>[] = [];
for (const pk of pks) {
const row = this.lsm.get(`${tableName}:${pk}`);
const row = await this.lsm.get(`${tableName}:${pk}`);
if (row) rows.push({ ...row, [pkCol]: pk });
}
return rows;