fix(P0): SSTable rangeScan 尾块漏读 — 索引块记录块尾 key 但二分按块首语义定位,endKey 落在块尾时下一块被排除导致二级索引查询丢数据(5万行丢106~771条);endBlockIdx 多扫一块 + 4 个 sstable 边界回归 + 重建二级索引大数据量回归(含崩溃恢复)。附带:checkpoint 同步 flush 二级索引 LSM(P1)、kv 后端页面化 + SharedMemoryBackend chunk 化、生产负载验证测试
This commit is contained in:
@@ -252,7 +252,17 @@ export class AriaEngine implements IStorageEngine {
|
||||
getBufferedBytes: () => this.wal.getBufferedBytes(),
|
||||
getBufferedCount: () => this.wal.getBufferedCount(),
|
||||
} as unknown as WAL,
|
||||
{ flushAll: async () => { await this.lsm.flush(); } } as any,
|
||||
{
|
||||
flushAll: async () => {
|
||||
// v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM —
|
||||
// 此前只 flush 主 LSM,checkpoint 截断 WAL 后崩溃时索引 memtable 未落盘、
|
||||
// WAL 为空跳过重建 → 二级索引静默丢失最后一批条目(生产数据一致性问题)
|
||||
await this.lsm.flush();
|
||||
for (const idxLsm of this.secondaryIndexes.values()) {
|
||||
await idxLsm.flush();
|
||||
}
|
||||
},
|
||||
} as any,
|
||||
this.config.checkpointInterval,
|
||||
this.config.walSizeThreshold,
|
||||
);
|
||||
@@ -1471,11 +1481,13 @@ export class AriaEngine implements IStorageEngine {
|
||||
};
|
||||
}
|
||||
|
||||
/** v0.4.5: 是否启用页面化物理存储(默认 OPFS 后端启用,显式配置可覆盖) */
|
||||
/** v0.4.5: 是否启用页面化物理存储(默认 OPFS / KVStore 后端启用,显式配置可覆盖) */
|
||||
private isPageStorage(): boolean {
|
||||
if (this.config.pageStorage === true) return true;
|
||||
if (this.config.pageStorage === false) return false;
|
||||
return this.config.storageBackend === 'opfs';
|
||||
// v0.6.1: kv 后端同样页面化 — 整 value SSTable(4MB)超过 1MB 页面缓存时
|
||||
// 每次读取全量重载;拆 4KB 页面后缓存按页命中,大数据量读放大消除
|
||||
return this.config.storageBackend === 'opfs' || this.config.storageBackend === 'kv';
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
|
||||
@@ -110,7 +110,11 @@ export class SSTableReader {
|
||||
): void {
|
||||
if (this.indexEntries.length === 0) return;
|
||||
const startBlockIdx = Math.max(0, this.locateBlockGE(startKey));
|
||||
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey));
|
||||
// v0.6.1-fix(P0): 索引键是"块内最后一个 key"(builder 约定),
|
||||
// locateBlockLE 返回最后一个 tail <= endKey 的块,但下一个块(tail > endKey)
|
||||
// 可能仍包含 < endKey 的条目(如末块 t5:k49425..k49995 尾 key 是 t6 前缀)→
|
||||
// 被排除导致范围扫描漏读尾部数据。多扫一个块,条目级过滤保证不丢。
|
||||
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1);
|
||||
if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx) return;
|
||||
|
||||
const lenSize = this.lenFieldSize();
|
||||
|
||||
@@ -11,12 +11,15 @@
|
||||
|
||||
import type { IStorageBackend } from '../aria/store/backend';
|
||||
|
||||
/** 全局注册表:dbName → key → ArrayBuffer(跨实例共享,模拟持久化) */
|
||||
const registry = new Map<string, Map<string, ArrayBuffer>>();
|
||||
/** 全局注册表:dbName → key → chunk 列表(跨实例共享,模拟持久化) */
|
||||
const registry = new Map<string, Map<string, ArrayBuffer[]>>();
|
||||
|
||||
export class SharedMemoryBackend implements IStorageBackend {
|
||||
private dbName = '';
|
||||
private store: Map<string, ArrayBuffer> | null = null;
|
||||
/** 存储:key → chunk 列表(append O(1),read 时一次性拼接缓存) */
|
||||
private chunks: Map<string, ArrayBuffer[]> = new Map();
|
||||
/** 惰性拼接缓存(read 后缓存,write/append 失效) */
|
||||
private materialized: Map<string, ArrayBuffer> = new Map();
|
||||
|
||||
/** 清空全局注册表(测试隔离用) */
|
||||
static clearRegistry(): void {
|
||||
@@ -28,67 +31,87 @@ export class SharedMemoryBackend implements IStorageBackend {
|
||||
if (!registry.has(name)) {
|
||||
registry.set(name, new Map());
|
||||
}
|
||||
this.store = registry.get(name)!;
|
||||
// 从注册表恢复 chunks(持久化语义)
|
||||
this.chunks = registry.get(name)! as unknown as Map<string, ArrayBuffer[]>;
|
||||
this.materialized = new Map();
|
||||
}
|
||||
|
||||
/** close 不清除数据(持久化语义:重开同名库数据仍在) */
|
||||
async close(): Promise<void> {
|
||||
this.store = null;
|
||||
this.chunks = new Map();
|
||||
this.materialized = new Map();
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.store !== null;
|
||||
return this.dbName !== '';
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
return this.store?.get(key) ?? null;
|
||||
if (this.materialized.has(key)) return this.materialized.get(key)!;
|
||||
const list = this.chunks.get(key);
|
||||
if (!list || list.length === 0) return null;
|
||||
if (list.length === 1) {
|
||||
this.materialized.set(key, list[0]);
|
||||
return list[0];
|
||||
}
|
||||
const total = list.reduce((s2, c) => s2 + c.byteLength, 0);
|
||||
const combined = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of list) {
|
||||
combined.set(new Uint8Array(c), off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
const buf = combined.buffer as ArrayBuffer;
|
||||
this.materialized.set(key, buf);
|
||||
return buf;
|
||||
}
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
this.store?.set(key, data);
|
||||
this.chunks.set(key, [data]);
|
||||
this.materialized.set(key, data);
|
||||
}
|
||||
|
||||
async append(key: string, data: ArrayBuffer): Promise<void> {
|
||||
if (!this.store) return;
|
||||
const existing = this.store.get(key);
|
||||
if (existing) {
|
||||
const combined = new Uint8Array(existing.byteLength + data.byteLength);
|
||||
combined.set(new Uint8Array(existing), 0);
|
||||
combined.set(new Uint8Array(data), existing.byteLength);
|
||||
this.store.set(key, combined.buffer as ArrayBuffer);
|
||||
// O(1) 追加:只记录 chunk,read 时惰性拼接
|
||||
const list = this.chunks.get(key);
|
||||
if (list) {
|
||||
list.push(data);
|
||||
} else {
|
||||
this.store.set(key, data);
|
||||
this.chunks.set(key, [data]);
|
||||
}
|
||||
this.materialized.delete(key);
|
||||
}
|
||||
|
||||
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
|
||||
if (!this.store) return;
|
||||
// 同步批量写入 = 原子(JS 单线程,无中间 await 点)
|
||||
for (const [key, data] of Object.entries(entries)) {
|
||||
this.store.set(key, data);
|
||||
this.chunks.set(key, [data]);
|
||||
this.materialized.set(key, data);
|
||||
}
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
this.store?.delete(key);
|
||||
this.chunks.delete(key);
|
||||
this.materialized.delete(key);
|
||||
}
|
||||
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
if (!this.store) return;
|
||||
for (const key of keys) {
|
||||
this.store.delete(key);
|
||||
this.chunks.delete(key);
|
||||
this.materialized.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
return this.store ? Array.from(this.store.keys()) : [];
|
||||
return Array.from(this.chunks.keys());
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
return this.store?.has(key) ?? false;
|
||||
return this.chunks.has(key);
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
this.store?.clear();
|
||||
this.chunks.clear();
|
||||
this.materialized.clear();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user