fix(P0): SSTable rangeScan 尾块漏读 — 索引块记录块尾 key 但二分按块首语义定位,endKey 落在块尾时下一块被排除导致二级索引查询丢数据(5万行丢106~771条);endBlockIdx 多扫一块 + 4 个 sstable 边界回归 + 重建二级索引大数据量回归(含崩溃恢复)。附带:checkpoint 同步 flush 二级索引 LSM(P1)、kv 后端页面化 + SharedMemoryBackend chunk 化、生产负载验证测试
CI / test (20.x) (push) Successful in 24m55s
CI / test (22.x) (push) Successful in 16m19s
CI / test (24.x) (push) Successful in 20m55s
CI / e2e (push) Successful in 9m54s
CI / test (18.x) (push) Successful in 26m38s

This commit is contained in:
thzxx
2026-08-10 18:18:50 +08:00
parent 25bab0f9aa
commit 34225a86b3
9 changed files with 579 additions and 38 deletions
+15 -3
View File
@@ -252,7 +252,17 @@ export class AriaEngine implements IStorageEngine {
getBufferedBytes: () => this.wal.getBufferedBytes(),
getBufferedCount: () => this.wal.getBufferedCount(),
} as unknown as WAL,
{ flushAll: async () => { await this.lsm.flush(); } } as any,
{
flushAll: async () => {
// v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM —
// 此前只 flush 主 LSMcheckpoint 截断 WAL 后崩溃时索引 memtable 未落盘、
// WAL 为空跳过重建 → 二级索引静默丢失最后一批条目(生产数据一致性问题)
await this.lsm.flush();
for (const idxLsm of this.secondaryIndexes.values()) {
await idxLsm.flush();
}
},
} as any,
this.config.checkpointInterval,
this.config.walSizeThreshold,
);
@@ -1471,11 +1481,13 @@ export class AriaEngine implements IStorageEngine {
};
}
/** v0.4.5: 是否启用页面化物理存储(默认 OPFS 后端启用,显式配置可覆盖) */
/** v0.4.5: 是否启用页面化物理存储(默认 OPFS / KVStore 后端启用,显式配置可覆盖) */
private isPageStorage(): boolean {
if (this.config.pageStorage === true) return true;
if (this.config.pageStorage === false) return false;
return this.config.storageBackend === 'opfs';
// v0.6.1: kv 后端同样页面化 — 整 value SSTable4MB)超过 1MB 页面缓存时
// 每次读取全量重载;拆 4KB 页面后缓存按页命中,大数据量读放大消除
return this.config.storageBackend === 'opfs' || this.config.storageBackend === 'kv';
}
// =======================================================================
+5 -1
View File
@@ -110,7 +110,11 @@ export class SSTableReader {
): void {
if (this.indexEntries.length === 0) return;
const startBlockIdx = Math.max(0, this.locateBlockGE(startKey));
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey));
// v0.6.1-fix(P0): 索引键是"块内最后一个 key"builder 约定),
// locateBlockLE 返回最后一个 tail <= endKey 的块,但下一个块(tail > endKey
// 可能仍包含 < endKey 的条目(如末块 t5:k49425..k49995 尾 key 是 t6 前缀)→
// 被排除导致范围扫描漏读尾部数据。多扫一个块,条目级过滤保证不丢。
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1);
if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx) return;
const lenSize = this.lenFieldSize();
+47 -24
View File
@@ -11,12 +11,15 @@
import type { IStorageBackend } from '../aria/store/backend';
/** 全局注册表:dbName → key → ArrayBuffer(跨实例共享,模拟持久化) */
const registry = new Map<string, Map<string, ArrayBuffer>>();
/** 全局注册表:dbName → key → chunk 列表(跨实例共享,模拟持久化) */
const registry = new Map<string, Map<string, ArrayBuffer[]>>();
export class SharedMemoryBackend implements IStorageBackend {
private dbName = '';
private store: Map<string, ArrayBuffer> | null = null;
/** 存储:key → chunk 列表(append O(1)read 时一次性拼接缓存) */
private chunks: Map<string, ArrayBuffer[]> = new Map();
/** 惰性拼接缓存(read 后缓存,write/append 失效) */
private materialized: Map<string, ArrayBuffer> = new Map();
/** 清空全局注册表(测试隔离用) */
static clearRegistry(): void {
@@ -28,67 +31,87 @@ export class SharedMemoryBackend implements IStorageBackend {
if (!registry.has(name)) {
registry.set(name, new Map());
}
this.store = registry.get(name)!;
// 从注册表恢复 chunks(持久化语义)
this.chunks = registry.get(name)! as unknown as Map<string, ArrayBuffer[]>;
this.materialized = new Map();
}
/** close 不清除数据(持久化语义:重开同名库数据仍在) */
async close(): Promise<void> {
this.store = null;
this.chunks = new Map();
this.materialized = new Map();
}
isOpen(): boolean {
return this.store !== null;
return this.dbName !== '';
}
async read(key: string): Promise<ArrayBuffer | null> {
return this.store?.get(key) ?? null;
if (this.materialized.has(key)) return this.materialized.get(key)!;
const list = this.chunks.get(key);
if (!list || list.length === 0) return null;
if (list.length === 1) {
this.materialized.set(key, list[0]);
return list[0];
}
const total = list.reduce((s2, c) => s2 + c.byteLength, 0);
const combined = new Uint8Array(total);
let off = 0;
for (const c of list) {
combined.set(new Uint8Array(c), off);
off += c.byteLength;
}
const buf = combined.buffer as ArrayBuffer;
this.materialized.set(key, buf);
return buf;
}
async write(key: string, data: ArrayBuffer): Promise<void> {
this.store?.set(key, data);
this.chunks.set(key, [data]);
this.materialized.set(key, data);
}
async append(key: string, data: ArrayBuffer): Promise<void> {
if (!this.store) return;
const existing = this.store.get(key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + data.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(data), existing.byteLength);
this.store.set(key, combined.buffer as ArrayBuffer);
// O(1) 追加:只记录 chunkread 时惰性拼接
const list = this.chunks.get(key);
if (list) {
list.push(data);
} else {
this.store.set(key, data);
this.chunks.set(key, [data]);
}
this.materialized.delete(key);
}
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
if (!this.store) return;
// 同步批量写入 = 原子(JS 单线程,无中间 await 点)
for (const [key, data] of Object.entries(entries)) {
this.store.set(key, data);
this.chunks.set(key, [data]);
this.materialized.set(key, data);
}
}
async delete(key: string): Promise<void> {
this.store?.delete(key);
this.chunks.delete(key);
this.materialized.delete(key);
}
async deleteMany(keys: string[]): Promise<void> {
if (!this.store) return;
for (const key of keys) {
this.store.delete(key);
this.chunks.delete(key);
this.materialized.delete(key);
}
}
async listKeys(): Promise<string[]> {
return this.store ? Array.from(this.store.keys()) : [];
return Array.from(this.chunks.keys());
}
async exists(key: string): Promise<boolean> {
return this.store?.has(key) ?? false;
return this.chunks.has(key);
}
async clear(): Promise<void> {
this.store?.clear();
this.chunks.clear();
this.materialized.clear();
}
}