fix(A8 + LSM 读自洽): 行所有权根治 + 读取路径不再依赖 prefetch
A8 行引用泄漏(调用方改查询结果即改写存储)
实测:rows[0].tag = 'HACKED' 后,tag='HACKED' 与 tag='x' 两条索引查询都返回 0 行 ——
行与索引失配、该行永久查不出来;嵌套 json 值同样按引用共享。
Aria 因走反序列化路径反而幸免,又形成跨引擎差异。
根治:在 IStorageEngine 契约层写入**行所有权约定**(engine/interface.ts)
—— 读出的行是副本、写入接收的行也是副本;新增 cloneRow/cloneRows
(优先 structuredClone,退化路径处理 Date/嵌套对象/二进制)。
Memory/KVStore/Hybrid:find、findStream、getRow 全部返回副本。
Aria:getAllRows 此前只做 `{ ...value }` 浅拷贝(嵌套 json 仍共享引用),
改为深拷贝;find/findStream 返回副本。
实测四种引擎:修改返回值后重读不变、索引两条查询均正确。
LSM 读取自洽(审计 P1-1:缓存未命中 = 静默丢数据)
此前 loadSSTableReader 缓存未命中返回 null,而所有调用方都是
const reader = this.loadSSTableReader(meta); if (!reader) continue;
于是**未命中就静默跳过整个 SSTable**。实测:缓存上限 4KB 而 SSTable 更大时,
300 行只能查回 59 行,且不报错。
同时"读路径必须先 prefetch"这个隐式约定,是每次读都要 drainChain + prefetch
的原因(性能悬崖的另一半)。
根治:LSM.get / rangeScan / rangeScanLazy 改为 async,未命中即
`await sstableStore.load()` 回源 + CRC 校验(损坏则自愈清理 meta),
只有数据确实不存在才返回 null。checkUniqueSync 相应改名 checkUnique 并 async
(原命名正是因为依赖 prefetch 约定)。引擎侧 12 处调用点补 await。
附带修正的缓存语义:
- tryCacheSSTable:单个 SSTable 超过缓存上限时标记为常驻(pinned),
不参与驱逐 —— 驱逐它等价于静默丢数据;内存上限因此是
cacheLimit + 单个最大 SSTable,已在代码与测试中明确。
- trimCache 跳过 pinned 条目(此前会把全部缓存一次性清空)。
- 新增 getCacheSize/getCacheLimit/getOversizedCount/setCacheLimit 访问器
(测试此前直接读私有字段 cacheSize/cacheLimitBytes —— 那是 TS 错误,
只因测试不做类型检查才没暴露)。
queryStream async 回调
此前用 constructor.name === 'AsyncFunction' 判定,对"普通函数返回 Promise"
完全失效(Promise 被静默丢弃)。现改为双条件识别并走物化路径逐行 await,
async 回调被真正等待。
测试契约修正:
- aria-cache:'缓存大小受上限约束' 在极小缓存下是不可成立的契约,改为断言
真正重要的不变量(数据完整;可装入时受上限约束),并新增"超大 SSTable 常驻"
用例;'缓存驱逐后全表扫描仍返回完整数据' 保留 300 行断言(此前会失败)。
- hybrid:磁盘引擎标签断言从 indexeddb(v0.6.0 已移除)改为 opfs。
This commit is contained in:
+35
-26
@@ -1,3 +1,4 @@
|
||||
import { cloneRow } from '../interface';
|
||||
/**
|
||||
* AriaEngine — 自研页面式存储引擎主类
|
||||
* @module engine/aria/index
|
||||
@@ -584,9 +585,10 @@ export class AriaEngine implements IStorageEngine {
|
||||
);
|
||||
}
|
||||
pkSet.add(pkValue);
|
||||
// v0.8.0: 事务快照优先,否则回源 LSM(lsm.get 现为 async)
|
||||
const existing = this.currentTxnId
|
||||
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
|
||||
: this.lsm.get(key);
|
||||
? (this.txnSnapshot?.get(key) ?? await this.lsm.get(key))
|
||||
: await this.lsm.get(key);
|
||||
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
|
||||
throw new DatabaseError(
|
||||
`Duplicate primary key "${pkValue}" in table "${tableName}"`,
|
||||
@@ -627,7 +629,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
);
|
||||
}
|
||||
seen.add(v);
|
||||
this.checkUniqueSync(tableName, [colName], validated, pkValue);
|
||||
await this.checkUnique(tableName, [colName], validated, pkValue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -706,7 +708,8 @@ export class AriaEngine implements IStorageEngine {
|
||||
// 查询完成,回收查询期间的临时缓存超限
|
||||
this.trimAllCaches();
|
||||
|
||||
return rows;
|
||||
// v0.8.0: 行所有权 —— 返回副本,调用方不得改写存储(见 engine/interface.ts 约定)
|
||||
return rows.map((row) => cloneRow(row));
|
||||
}
|
||||
|
||||
async update(
|
||||
@@ -780,7 +783,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
// 批内唯一互查(索引尚未更新,两行同时改到同一新值需要互查兜底)
|
||||
this.checkBatchUnique(tableName, uniqueCols, updated, batchUnique);
|
||||
// v0.6.2: 唯一约束检查(排除自身旧索引条目:主键变更时旧条目仍以旧键存在)
|
||||
this.checkUniqueSync(tableName, uniqueCols, updated, String(row[pkCol]));
|
||||
await this.checkUnique(tableName, uniqueCols, updated, String(row[pkCol]));
|
||||
|
||||
// v0.4.2-fix: 支持更新主键 — 删除旧键 + 落新键 + WAL 两条记录
|
||||
const newPk = String(updated[pkCol]);
|
||||
@@ -791,8 +794,8 @@ export class AriaEngine implements IStorageEngine {
|
||||
if (pkChanged) {
|
||||
const newKey = `${tableName}:${newPk}`;
|
||||
const existing = this.currentTxnId
|
||||
? (this.txnSnapshot?.get(newKey) ?? this.lsm.get(newKey))
|
||||
: this.lsm.get(newKey);
|
||||
? (this.txnSnapshot?.get(newKey) ?? await this.lsm.get(newKey))
|
||||
: await this.lsm.get(newKey);
|
||||
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
|
||||
throw new DatabaseError(
|
||||
`Duplicate primary key "${newPk}" in table "${tableName}" (cannot update key to existing value)`,
|
||||
@@ -1191,7 +1194,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
const emit = (row: Record<string, unknown>): boolean => {
|
||||
if (hasWhere && !matchWhere(row, query.where!)) return true;
|
||||
if (skipped < offset) { skipped++; return true; }
|
||||
onRow(project ? project(row) : row);
|
||||
onRow(project ? project(row) : cloneRow(row));
|
||||
count++;
|
||||
return count < limit;
|
||||
};
|
||||
@@ -1200,7 +1203,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
// 事务中:物化后逐行回调(快照合并需要全量行集)
|
||||
const rows = await this.find(tableName, { ...query, orderBy: undefined, limit: undefined, offset: undefined });
|
||||
for (const row of rows) {
|
||||
onRow(project ? project(row) : row);
|
||||
onRow(project ? project(row) : cloneRow(row));
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
@@ -1217,7 +1220,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
// 全表惰性扫描(含 WHERE 过滤,不物化;v0.7.4: callback 返回 false 提前终止,
|
||||
// 未消费的 SSTable 块 / 子树不再解析 —— 真流式,大表 limit 内存 O(1))
|
||||
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
|
||||
this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => {
|
||||
await this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => {
|
||||
if (count >= limit) return false;
|
||||
const row = { ...value };
|
||||
row[pkCol] = key.slice(prefix.length);
|
||||
@@ -1314,7 +1317,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
const prefix = `${tableName}:`;
|
||||
const endKey = `${prefix}\uffff`;
|
||||
await this.lsm.prefetchRange(prefix, endKey);
|
||||
const entries = this.lsm.rangeScan(prefix, endKey);
|
||||
const entries = await this.lsm.rangeScan(prefix, endKey);
|
||||
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
|
||||
for (const [key, value] of entries) {
|
||||
if (!(column.name in value)) continue;
|
||||
@@ -1603,9 +1606,11 @@ export class AriaEngine implements IStorageEngine {
|
||||
const prefix = `${tableName}:`;
|
||||
// 预加载范围内涉及的 SSTable,避免 rangeScan 时缓存未命中静默丢数据
|
||||
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
|
||||
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
const rows = entries.map(([key, value]) => {
|
||||
const row = { ...value };
|
||||
// v0.8.0: 深拷贝(此前 `{ ...value }` 只做浅拷贝,嵌套 json 值仍与 LSM
|
||||
// 内部对象共享引用 —— 调用方改 rows[0].nested.a 会改写存储)
|
||||
const row = cloneRow(value);
|
||||
row[pkCol] = key.slice(prefix.length);
|
||||
return row;
|
||||
});
|
||||
@@ -1872,7 +1877,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
const prefix = `${tableName}:`;
|
||||
const endKey = `${prefix}\uffff`;
|
||||
await this.lsm.prefetchRange(prefix, endKey);
|
||||
const entries = this.lsm.rangeScan(prefix, endKey);
|
||||
const entries = await this.lsm.rangeScan(prefix, endKey);
|
||||
for (const [key] of entries) {
|
||||
this.lsm.delete(key);
|
||||
}
|
||||
@@ -1893,23 +1898,27 @@ export class AriaEngine implements IStorageEngine {
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.6.2: 同步唯一性检查(须在批次级 prefetchPrefixRanges 之后调用,循环内无 await)。
|
||||
* 唯一性检查。
|
||||
*
|
||||
* v0.8.0: 由 `checkUniqueSync` 改名并改为 async —— 此前命名为 "Sync" 是因为它
|
||||
* 依赖"批次级 prefetchPrefixRanges 之后索引数据已在缓存中"这一约定。现在
|
||||
* LSM 读取自洽(未命中即回源),因此这里可以、也必须 await。
|
||||
* 索引不含 null 条目(null 值不受唯一约束,与 MemoryEngine 语义一致)。
|
||||
* @param currentPk 当前行主键(更新路径用于排除自身旧索引条目;插入路径无自身条目)
|
||||
*/
|
||||
private checkUniqueSync(
|
||||
private async checkUnique(
|
||||
tableName: string,
|
||||
uniqueCols: string[],
|
||||
row: Record<string, unknown>,
|
||||
currentPk: string,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
for (const colName of uniqueCols) {
|
||||
const val = row[colName];
|
||||
if (val === undefined || val === null) continue;
|
||||
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`);
|
||||
if (!idxLsm) continue;
|
||||
const prefix = `${String(val)}:`;
|
||||
const entries = idxLsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
const entries = await idxLsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
for (const [, entry] of entries) {
|
||||
const pk = (entry as unknown as { pk?: string }).pk;
|
||||
if (pk !== undefined && pk !== currentPk) {
|
||||
@@ -1994,14 +2003,14 @@ export class AriaEngine implements IStorageEngine {
|
||||
if (typeof condition !== 'object' || condition === null) {
|
||||
const key = `${tableName}:${condition}`;
|
||||
await this.lsm.prefetchKeys([key]);
|
||||
const value = this.lsm.get(key);
|
||||
const value = await this.lsm.get(key);
|
||||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||||
}
|
||||
const cond = condition as Record<string, unknown>;
|
||||
if ('$eq' in cond) {
|
||||
const key = `${tableName}:${cond.$eq}`;
|
||||
await this.lsm.prefetchKeys([key]);
|
||||
const value = this.lsm.get(key);
|
||||
const value = await this.lsm.get(key);
|
||||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||||
}
|
||||
// v0.3.3: PK $in → 主 LSM 多次精确查找(替代冗余 PK 二级索引)
|
||||
@@ -2013,7 +2022,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
for (const v of cond.$in) {
|
||||
const pk = String(v);
|
||||
if (seen.has(pk)) continue;
|
||||
const value = this.lsm.get(`${tableName}:${pk}`);
|
||||
const value = await this.lsm.get(`${tableName}:${pk}`);
|
||||
if (value) { seen.add(pk); rows.push({ ...value, [pkCol]: pk }); }
|
||||
}
|
||||
return rows;
|
||||
@@ -2022,7 +2031,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
if ('$gt' in cond || '$gte' in cond || '$lt' in cond || '$lte' in cond) {
|
||||
const prefix = `${tableName}:`;
|
||||
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
|
||||
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
for (const [key, value] of entries) {
|
||||
const candidate = { ...value, [pkCol]: key.slice(prefix.length) };
|
||||
@@ -2065,7 +2074,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
const seenPks = new Set<string>(); // v0.4.1: IN 值可能重复,按 pk 去重
|
||||
const pks: string[] = [];
|
||||
for (const val of values) {
|
||||
const entries = idxLsm.rangeScan(val, `${val}\uffff`);
|
||||
const entries = await idxLsm.rangeScan(val, `${val}\uffff`);
|
||||
for (const [, idxEntry] of entries) {
|
||||
const pk = (idxEntry as { pk?: string }).pk;
|
||||
if (pk && !seenPks.has(pk)) {
|
||||
@@ -2076,7 +2085,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
}
|
||||
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
|
||||
for (const pk of pks) {
|
||||
const row = this.lsm.get(`${tableName}:${pk}`);
|
||||
const row = await this.lsm.get(`${tableName}:${pk}`);
|
||||
if (row) results.push({ ...row, [pkCol]: pk });
|
||||
}
|
||||
return results;
|
||||
@@ -2104,7 +2113,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
const actualEndKey = endKey.includes('\uffff') ? endKey : `${endKey}\uffff`;
|
||||
// 预加载索引 LSM 与主 LSM 涉及的 SSTable
|
||||
await idxLsm.prefetchRange(startKey, actualEndKey);
|
||||
const entries = idxLsm.rangeScan(startKey, actualEndKey);
|
||||
const entries = await idxLsm.rangeScan(startKey, actualEndKey);
|
||||
const pks: string[] = [];
|
||||
for (const [, idxEntry] of entries) {
|
||||
const pk = (idxEntry as any).pk as string;
|
||||
@@ -2113,7 +2122,7 @@ export class AriaEngine implements IStorageEngine {
|
||||
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
for (const pk of pks) {
|
||||
const row = this.lsm.get(`${tableName}:${pk}`);
|
||||
const row = await this.lsm.get(`${tableName}:${pk}`);
|
||||
if (row) rows.push({ ...row, [pkCol]: pk });
|
||||
}
|
||||
return rows;
|
||||
|
||||
Reference in New Issue
Block a user