release: v0.6.0 — 完全移除 IndexedDB，自研 KVStore 事务存储引擎（多key原子写/快照日志恢复/CRC自愈）+ KVStoreEngine + 旧库迁移工具 + 10万级压力验证 + 崩溃注入e2e
This commit is contained in:
thzxx
@@ -8,7 +8,10 @@
|
||||
*/
|
||||
import { AriaEngine } from '../../src/engine/aria/index';
|
||||
import { createSchema } from '../../src/table/schema';
|
||||
import 'fake-indexeddb/auto';
|
||||
|
||||
import { installOPFSMock } from '../helpers/opfs-mock';
|
||||
|
||||
beforeEach(() => { installOPFSMock(new Map()); });
|
||||
|
||||
let idbCounter = 0;
|
||||
function uniqueDB(): string {
|
||||
@@ -21,7 +24,7 @@ const SCHEMA = () => createSchema('users', {
|
||||
age: { type: 'number' },
|
||||
});
|
||||
|
||||
/** 篡改 MemoryBackend 中指定 key 的一个字节 */
|
||||
/** 篡改指定 pg_ 页面文件的一个字节 */
|
||||
async function corruptKey(engine: AriaEngine, key: string, byteOffset: number): Promise<void> {
|
||||
const backend = (engine as any).backend;
|
||||
const raw = await backend.read(key);
|
||||
@@ -31,16 +34,24 @@ async function corruptKey(engine: AriaEngine, key: string, byteOffset: number):
|
||||
await backend.write(key, buf.buffer as ArrayBuffer);
|
||||
}
|
||||
|
||||
/** 列出 backend 中所有 sst_ 前缀 key */
|
||||
/** 列出全部 SSTable 的页面文件 key（页面化存储：pg_ 前缀） */
|
||||
async function listSSTKeys(engine: AriaEngine): Promise<string[]> {
|
||||
const backend = (engine as any).backend;
|
||||
const keys = await backend.listKeys();
|
||||
return (keys as string[]).filter((k) => k.startsWith('sst_'));
|
||||
const keys = (await backend.listKeys()) as string[];
|
||||
return keys.filter((k) => k.startsWith('pg_'));
|
||||
}
|
||||
|
||||
/** 读取主 LSM 的 SSTable meta 列表 */
|
||||
async function listSSTMetas(engine: AriaEngine): Promise<{ id: number; pageIds?: number[] }[]> {
|
||||
const backend = (engine as any).backend;
|
||||
const raw = await backend.read('__aria_lsm_meta');
|
||||
if (!raw) return [];
|
||||
return JSON.parse(new TextDecoder().decode(raw)) as { id: number; pageIds?: number[] }[];
|
||||
}
|
||||
|
||||
describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
it('打开时 CRC 损坏的 SSTable 被清理，其余数据可读', async () => {
|
||||
const engine = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const dbName = uniqueDB();
|
||||
await engine.open(dbName, 1);
|
||||
await engine.createTable(SCHEMA());
|
||||
@@ -57,7 +68,7 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
await engine.close();
|
||||
|
||||
// 重新打开，篡改第一个 SSTable 文件的数据区
|
||||
const engine2 = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine2 = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
await engine2.open(dbName, 1);
|
||||
const sstKeys = await listSSTKeys(engine2);
|
||||
expect(sstKeys.length).toBeGreaterThanOrEqual(2);
|
||||
@@ -65,7 +76,7 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
|
||||
// 再次打开：损坏文件应被跳过并清理，打开不抛错
|
||||
await engine2.close();
|
||||
const engine3 = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine3 = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
await engine3.open(dbName, 1);
|
||||
|
||||
// 剩余未损坏文件的数据应可查询
|
||||
@@ -73,18 +84,20 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
expect(remaining.length).toBeGreaterThan(0);
|
||||
expect(remaining.length).toBeLessThan(200);
|
||||
|
||||
// 损坏文件已被清理（meta 移除）
|
||||
// 损坏文件已被清理（页面 + meta 移除）
|
||||
const afterKeys = await listSSTKeys(engine3);
|
||||
expect(afterKeys).not.toContain(sstKeys[0]);
|
||||
const lsm = (engine3 as any).lsm;
|
||||
const metas = await lsm.sstableStore.listMeta();
|
||||
expect(metas.some((m: { id: number }) => m.id === Number(sstKeys[0].replace('sst_', '')))).toBe(false);
|
||||
const metas = await listSSTMetas(engine3);
|
||||
// 被篡改页面所属的 SSTable meta 应被移除
|
||||
const victimMetas = await listSSTMetas(engine2);
|
||||
const victimMeta = victimMetas.find((m) => m.pageIds?.includes(Number(sstKeys[0].slice(3))));
|
||||
expect(metas.some((m) => m.id === victimMeta?.id)).toBe(false);
|
||||
|
||||
await engine3.close();
|
||||
});
|
||||
|
||||
it('打开时损坏全部 SSTable → 库仍可打开，数据为空但不崩溃', async () => {
|
||||
const engine = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const dbName = uniqueDB();
|
||||
await engine.open(dbName, 1);
|
||||
await engine.createTable(SCHEMA());
|
||||
@@ -98,14 +111,14 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
}
|
||||
await engine.close();
|
||||
|
||||
const engine2 = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine2 = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
await engine2.open(dbName, 1);
|
||||
for (const key of await listSSTKeys(engine2)) {
|
||||
await corruptKey(engine2, key, 16);
|
||||
}
|
||||
await engine2.close();
|
||||
|
||||
const engine3 = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine3 = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
// 不应抛 ARIA_OPEN_ERROR
|
||||
await engine3.open(dbName, 1);
|
||||
const rows3 = await engine3.find('users', { table: 'users' });
|
||||
@@ -114,7 +127,7 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
});
|
||||
|
||||
it('运行期 CRC 损坏 → 预加载不缓存损坏文件并清理（自愈）', async () => {
|
||||
const engine = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const dbName = uniqueDB();
|
||||
await engine.open(dbName, 1);
|
||||
await engine.createTable(SCHEMA());
|
||||
@@ -133,22 +146,23 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
const victim = sstKeys[sstKeys.length - 1]; // 篡改最新（memtable 已 flush 后的文件）
|
||||
await corruptKey(engine, victim, 100);
|
||||
|
||||
// 缓存里已有该文件（flush 时缓存）→ 先清缓存模拟运行期重载
|
||||
// 缓存里已有该文件（flush 时缓存）→ 清 LSM 层与 BufferPool 页面缓存模拟运行期磁盘损坏
|
||||
const lsm = (engine as any).lsm;
|
||||
lsm.sstableCache.delete(Number(victim.replace('sst_', '')));
|
||||
lsm.sstableCache.clear();
|
||||
await (engine as any).bufferPool.clear();
|
||||
|
||||
// 查询触发 prefetchRange → preloadSSTable 发现 CRC 失败 → 清理 + 不缓存
|
||||
const remaining = await engine.find('users', { table: 'users' });
|
||||
expect(remaining.length).toBeLessThan(100);
|
||||
expect(lsm.sstableCache.has(Number(victim.replace('sst_', '')))).toBe(false);
|
||||
expect(lsm.sstableCache.has(Number(victim.slice(3)))).toBe(false);
|
||||
const metas = await lsm.sstableStore.listMeta();
|
||||
expect(metas.some((m: { id: number }) => m.id === Number(victim.replace('sst_', '')))).toBe(false);
|
||||
expect(metas.some((m: { id: number }) => m.id === Number(victim.slice(3)))).toBe(false);
|
||||
|
||||
await engine.close();
|
||||
});
|
||||
|
||||
it('旧版无校验文件（checksum=0）在 LSM 中正常加载', async () => {
|
||||
const engine = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const dbName = uniqueDB();
|
||||
await engine.open(dbName, 1);
|
||||
await engine.createTable(SCHEMA());
|
||||
@@ -159,7 +173,7 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
await (engine as any).lsm.flush();
|
||||
await engine.close();
|
||||
|
||||
const engine2 = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine2 = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
await engine2.open(dbName, 1);
|
||||
const sstKeys = await listSSTKeys(engine2);
|
||||
expect(sstKeys.length).toBe(1);
|
||||
@@ -171,7 +185,7 @@ describe('AriaEngine — SSTable CRC 损坏检测（集成）', () => {
|
||||
await engine2.close();
|
||||
|
||||
// 重开：checksum=0 跳过校验，数据完整可读
|
||||
const engine3 = new AriaEngine({ storageBackend: 'indexeddb', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
const engine3 = new AriaEngine({ storageBackend: 'opfs', memtableSizeThreshold: 64 * 1024 * 1024 });
|
||||
await engine3.open(dbName, 1);
|
||||
const rows3 = await engine3.find('users', { table: 'users' });
|
||||
expect(rows3.length).toBe(2);
|
||||
|
||||
Reference in New Issue
Block a user