release: v0.6.0 — 完全移除 IndexedDB，自研 KVStore 事务存储引擎（多key原子写/快照日志恢复/CRC自愈）+ KVStoreEngine + 旧库迁移工具 + 10万级压力验证 + 崩溃注入e2e
This commit is contained in:
thzxx
@@ -0,0 +1,394 @@
|
||||
/**
|
||||
* KVStore — 自研 KV 事务存储引擎（替代 IndexedDB）
|
||||
* @module engine/kvstore/index
|
||||
*
|
||||
* v0.6.0: 在浏览器文件系统（OPFS）之上实现 IndexedDB 级能力：
|
||||
* - 多 key 原子事务：putMany/deleteMany 写入单条日志记录（单文件 COW 原子追加）→
|
||||
* 崩溃时记录全有或全无（IndexedDB 事务同等的原子性，但完全自研）
|
||||
* - 持久化与崩溃恢复：快照（checkpoint）+ 追加日志（WAL 式），两阶段恢复
|
||||
* - 自愈：快照损坏回退全量日志重放；日志损坏截断至损坏处（丢弃未确认尾部）
|
||||
* - 容错时序：checkpoint = 写快照 → 写 meta → 清空日志（meta 先于截断，
|
||||
* 任何崩溃窗口数据不丢）
|
||||
*
|
||||
* 介质层为 IStorageBackend（OPFSBackend / SharedMemoryBackend）：
|
||||
* - 浏览器：自动选择 OPFS（navigator.storage）
|
||||
* - Node/测试：SharedMemoryBackend（跨实例共享，模拟持久化）
|
||||
*
|
||||
* 可靠性设计：
|
||||
* - 所有写操作与 checkpoint 经内部串行队列（快照与日志水位一致，无交错窗口）
|
||||
* - 日志记录与快照均有标准 CRC-32 校验
|
||||
* - 内存索引为热路径（get O(1)），checkpoint 后日志截断
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from '../aria/store/backend';
|
||||
import { OPFSBackend } from '../aria/store/opfs_backend';
|
||||
import { SharedMemoryBackend } from './shared_memory_medium';
|
||||
import { DatabaseError } from '../../constants';
|
||||
import { encodeLogRecord, parseLogRecords, KVLogOp } from './log';
|
||||
import { encodeSnapshot, decodeSnapshot } from './snapshot';
|
||||
import { crc32 } from '../aria/crc32';
|
||||
|
||||
/** 存储键 */
|
||||
const LOG_KEY = '__kv_log';
|
||||
const SNAPSHOT_KEY = '__kv_snapshot';
|
||||
const META_KEY = '__kv_meta';
|
||||
|
||||
/** checkpoint 自动触发阈值（日志字节数，0=不自动） */
|
||||
const DEFAULT_CHECKPOINT_THRESHOLD = 16 * 1024 * 1024;
|
||||
|
||||
interface KVStoreMeta {
|
||||
/** 当前日志水位（快照内嵌；无快照时 0） */
|
||||
seq: number;
|
||||
}
|
||||
|
||||
function defaultMedium(): IStorageBackend {
|
||||
const nav = (globalThis as { navigator?: { storage?: { getDirectory?: unknown } } }).navigator;
|
||||
if (typeof nav !== 'undefined' && nav.storage && typeof nav.storage.getDirectory === 'function') {
|
||||
return new OPFSBackend();
|
||||
}
|
||||
return new SharedMemoryBackend();
|
||||
}
|
||||
|
||||
export class KVStore {
|
||||
private medium: IStorageBackend;
|
||||
private dbName = '';
|
||||
private opened = false;
|
||||
|
||||
/** 内存索引（热路径权威视图） */
|
||||
private index = new Map<string, ArrayBuffer>();
|
||||
/** 日志水位（最后一条已应用日志记录序号） */
|
||||
private seq = 0;
|
||||
/** 日志累计字节数（checkpoint 阈值） */
|
||||
private logBytes = 0;
|
||||
/** checkpoint 自动触发阈值（字节） */
|
||||
private checkpointThreshold: number;
|
||||
|
||||
/** 写操作串行队列（checkpoint 与写入无交错窗口） */
|
||||
private opQueue: Promise<unknown> = Promise.resolve();
|
||||
/** 最近一次后台操作失败（checkpoint 时报告） */
|
||||
private lastBackgroundError: unknown = null;
|
||||
|
||||
constructor(medium?: IStorageBackend, checkpointThreshold: number = DEFAULT_CHECKPOINT_THRESHOLD) {
|
||||
this.medium = medium ?? defaultMedium();
|
||||
this.checkpointThreshold = checkpointThreshold;
|
||||
}
|
||||
|
||||
/** 底层介质（测试/诊断用） */
|
||||
getMedium(): IStorageBackend {
|
||||
return this.medium;
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.opened;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 生命周期
|
||||
// =======================================================================
|
||||
|
||||
/** 打开（加载快照 + 重放日志） */
|
||||
async open(dbName: string): Promise<void> {
|
||||
if (this.opened) return;
|
||||
this.dbName = dbName;
|
||||
await this.medium.open(dbName);
|
||||
this.index = new Map();
|
||||
this.seq = 0;
|
||||
this.logBytes = 0;
|
||||
|
||||
// 1. 读 meta（可能缺失/损坏）
|
||||
let metaSeq = 0;
|
||||
const metaRaw = await this.medium.read(META_KEY);
|
||||
if (metaRaw) {
|
||||
try {
|
||||
const meta = JSON.parse(new TextDecoder().decode(metaRaw)) as KVStoreMeta;
|
||||
metaSeq = Number(meta.seq) || 0;
|
||||
} catch { /* meta 损坏：回退全量日志 */ }
|
||||
}
|
||||
|
||||
// 2. 加载快照（损坏则全量日志重放）
|
||||
let snapshotSeq = 0;
|
||||
const snapshotRaw = await this.medium.read(SNAPSHOT_KEY);
|
||||
if (snapshotRaw) {
|
||||
const snap = decodeSnapshot(new Uint8Array(snapshotRaw));
|
||||
if (snap) {
|
||||
this.index = new Map(snap.entries);
|
||||
this.seq = snap.seq;
|
||||
snapshotSeq = snap.seq;
|
||||
} else {
|
||||
// 快照损坏：从空索引 + 全量日志重放
|
||||
this.index = new Map();
|
||||
this.seq = 0;
|
||||
snapshotSeq = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 重放日志（seq > 快照水位的记录）
|
||||
const logRaw = await this.medium.read(LOG_KEY);
|
||||
if (logRaw && logRaw.byteLength > 0) {
|
||||
const log = new Uint8Array(logRaw);
|
||||
const baseSeq = Math.max(metaSeq, snapshotSeq);
|
||||
const corruptOffsets: number[] = [];
|
||||
const applied = parseLogRecords(log, (record) => {
|
||||
if (record.seq <= baseSeq) return; // 快照已包含，跳过（幂等）
|
||||
this.applyRecord(record.entries);
|
||||
this.seq = record.seq;
|
||||
}, (offset) => {
|
||||
corruptOffsets.push(offset);
|
||||
return true; // 记录损坏位置后停止（日志是顺序流，无法跳过继续）
|
||||
});
|
||||
if (applied > 0 || corruptOffsets.length > 0) {
|
||||
this.logBytes = log.byteLength;
|
||||
}
|
||||
if (corruptOffsets.length > 0) {
|
||||
// 损坏尾部：截断日志（丢弃未确认记录），下次 checkpoint 落盘
|
||||
await this.truncateLog();
|
||||
}
|
||||
}
|
||||
|
||||
this.opened = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.6.0: 从介质重新加载（多标签页同步/外部写入可见用）。
|
||||
* KVStore 的内存索引非跨实例共享，重新 open 读取介质最新数据。
|
||||
*/
|
||||
async reload(): Promise<void> {
|
||||
if (!this.opened) return;
|
||||
try { await this.opQueue; } catch { /* ignore */ }
|
||||
this.index = new Map();
|
||||
this.seq = 0;
|
||||
this.logBytes = 0;
|
||||
this.opened = false;
|
||||
await this.open(this.dbName);
|
||||
}
|
||||
|
||||
/** 关闭（不丢弃数据；下次 open 同名库恢复） */
|
||||
async close(): Promise<void> {
|
||||
if (!this.opened) return;
|
||||
// 排空写队列
|
||||
try { await this.opQueue; } catch { /* 写失败已返回 */ }
|
||||
await this.medium.close();
|
||||
this.index.clear();
|
||||
this.seq = 0;
|
||||
this.logBytes = 0;
|
||||
this.opened = false;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 读写（内存热路径）
|
||||
// =======================================================================
|
||||
|
||||
async get(key: string): Promise<ArrayBuffer | null> {
|
||||
return this.index.get(key) ?? null;
|
||||
}
|
||||
|
||||
async getAll(): Promise<[string, ArrayBuffer][]> {
|
||||
return Array.from(this.index.entries());
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
return Array.from(this.index.keys());
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
return this.index.has(key);
|
||||
}
|
||||
|
||||
size(): number {
|
||||
return this.index.size;
|
||||
}
|
||||
|
||||
getSeq(): number {
|
||||
return this.seq;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 写入（原子事务）
|
||||
// =======================================================================
|
||||
|
||||
/** 单 key 写入（原子） */
|
||||
async put(key: string, value: ArrayBuffer): Promise<void> {
|
||||
await this.enqueue(async () => {
|
||||
await this.appendRecord({ [key]: value }, []);
|
||||
});
|
||||
}
|
||||
|
||||
/** 多 key 原子写入（单条日志记录，崩溃全有或全无） */
|
||||
async putMany(entries: Record<string, ArrayBuffer>): Promise<void> {
|
||||
if (Object.keys(entries).length === 0) return;
|
||||
await this.enqueue(async () => {
|
||||
await this.appendRecord(entries, []);
|
||||
});
|
||||
}
|
||||
|
||||
/** 单 key 删除（原子） */
|
||||
async delete(key: string): Promise<void> {
|
||||
await this.enqueue(async () => {
|
||||
await this.appendRecord({}, [key]);
|
||||
});
|
||||
}
|
||||
|
||||
/** 多 key 原子删除（单条日志记录） */
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
if (keys.length === 0) return;
|
||||
await this.enqueue(async () => {
|
||||
await this.appendRecord({}, keys);
|
||||
});
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 维护
|
||||
// =======================================================================
|
||||
|
||||
/** checkpoint：快照 → meta → 截断日志（时序保证任何崩溃窗口不丢数据） */
|
||||
async checkpoint(): Promise<void> {
|
||||
await this.enqueue(async () => {
|
||||
// 报告上次后台失败
|
||||
if (this.lastBackgroundError !== null) {
|
||||
const error = this.lastBackgroundError;
|
||||
this.lastBackgroundError = null;
|
||||
throw new DatabaseError('KVStore background write failed', 'KV_BACKGROUND_ERROR', error);
|
||||
}
|
||||
if (this.logBytes === 0 && this.index.size === 0) return;
|
||||
|
||||
// 1. 写快照（COW 原子）
|
||||
const snapBytes = encodeSnapshot(this.seq, this.index);
|
||||
await this.medium.write(SNAPSHOT_KEY, snapBytes.buffer as ArrayBuffer);
|
||||
// 2. 写 meta（指向新水位）
|
||||
const meta: KVStoreMeta = { seq: this.seq };
|
||||
await this.medium.write(META_KEY, new TextEncoder().encode(JSON.stringify(meta)).buffer);
|
||||
// 3. 截断日志（meta 已更新 → 截断安全）
|
||||
await this.truncateLog();
|
||||
});
|
||||
}
|
||||
|
||||
/** 清空全部数据（保留库本身） */
|
||||
async clear(): Promise<void> {
|
||||
await this.enqueue(async () => {
|
||||
await this.medium.clear();
|
||||
this.index.clear();
|
||||
this.seq = 0;
|
||||
this.logBytes = 0;
|
||||
// 写空 meta（下次 open 正常初始化）
|
||||
const meta: KVStoreMeta = { seq: 0 };
|
||||
await this.medium.write(META_KEY, new TextEncoder().encode(JSON.stringify(meta)).buffer);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 自愈：校验快照/日志完整性，清理损坏数据。
|
||||
* @returns 丢弃的损坏日志字节数（0 = 无损坏）
|
||||
*/
|
||||
async repair(): Promise<number> {
|
||||
return this.enqueue(async () => {
|
||||
let discarded = 0;
|
||||
// 1. 校验快照：损坏则删除（下次打开全量日志重放）
|
||||
const snapRaw = await this.medium.read(SNAPSHOT_KEY);
|
||||
if (snapRaw && !decodeSnapshot(new Uint8Array(snapRaw))) {
|
||||
await this.medium.delete(SNAPSHOT_KEY);
|
||||
discarded++;
|
||||
}
|
||||
// 2. 校验日志：损坏尾部截断
|
||||
const logRaw = await this.medium.read(LOG_KEY);
|
||||
if (logRaw && logRaw.byteLength > 0) {
|
||||
const log = new Uint8Array(logRaw);
|
||||
const validBytes = this.findValidLogLength(log);
|
||||
if (validBytes < log.byteLength) {
|
||||
discarded += log.byteLength - validBytes;
|
||||
const truncated = log.subarray(0, validBytes).slice().buffer as ArrayBuffer;
|
||||
await this.medium.write(LOG_KEY, truncated);
|
||||
}
|
||||
}
|
||||
return discarded;
|
||||
});
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 内部
|
||||
// =======================================================================
|
||||
|
||||
private enqueue<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const run = this.opQueue.then(fn, fn);
|
||||
this.opQueue = run.then(() => undefined, () => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
/** 追加一条日志记录并更新内存索引（队列内调用，无并发） */
|
||||
private async appendRecord(puts: Record<string, ArrayBuffer>, deletes: string[]): Promise<void> {
|
||||
this.seq++;
|
||||
const record = encodeLogRecord(this.seq, puts, deletes);
|
||||
try {
|
||||
// 日志追加：介质 append（真追加）或回退读+拼+写
|
||||
const data = record.buffer.slice(record.byteOffset, record.byteOffset + record.byteLength) as ArrayBuffer;
|
||||
if (typeof this.medium.append === 'function') {
|
||||
await this.medium.append(LOG_KEY, data);
|
||||
} else {
|
||||
const existing = await this.medium.read(LOG_KEY);
|
||||
if (existing) {
|
||||
const combined = new Uint8Array(existing.byteLength + data.byteLength);
|
||||
combined.set(new Uint8Array(existing), 0);
|
||||
combined.set(new Uint8Array(data), existing.byteLength);
|
||||
await this.medium.write(LOG_KEY, combined.buffer as ArrayBuffer);
|
||||
} else {
|
||||
await this.medium.write(LOG_KEY, data);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// 记录写入失败：内存索引不更新（原子性），记录后台错误
|
||||
this.seq--; // 回滚水位
|
||||
this.lastBackgroundError = error;
|
||||
throw new DatabaseError('KVStore log append failed', 'KV_LOG_ERROR', error);
|
||||
}
|
||||
|
||||
// 日志成功：更新内存索引（原子语义）
|
||||
for (const [key, value] of Object.entries(puts)) {
|
||||
this.index.set(key, value);
|
||||
}
|
||||
for (const key of deletes) {
|
||||
this.index.delete(key);
|
||||
}
|
||||
this.logBytes += record.byteLength;
|
||||
|
||||
// 自动 checkpoint（日志超阈值）
|
||||
if (this.checkpointThreshold > 0 && this.logBytes >= this.checkpointThreshold) {
|
||||
await this.medium.write(SNAPSHOT_KEY, encodeSnapshot(this.seq, this.index).buffer as ArrayBuffer);
|
||||
const meta: KVStoreMeta = { seq: this.seq };
|
||||
await this.medium.write(META_KEY, new TextEncoder().encode(JSON.stringify(meta)).buffer);
|
||||
await this.truncateLog();
|
||||
}
|
||||
}
|
||||
|
||||
/** 截断日志（清空文件） */
|
||||
private async truncateLog(): Promise<void> {
|
||||
try {
|
||||
await this.medium.write(LOG_KEY, new ArrayBuffer(0));
|
||||
} catch { /* 截断失败：下次 checkpoint 重试 */ }
|
||||
this.logBytes = 0;
|
||||
}
|
||||
|
||||
/** 应用记录条目到内存索引 */
|
||||
private applyRecord(entries: { op: KVLogOp; key: string; value: ArrayBuffer }[]): void {
|
||||
for (const e of entries) {
|
||||
if (e.op === KVLogOp.PUT) {
|
||||
this.index.set(e.key, e.value);
|
||||
} else {
|
||||
this.index.delete(e.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 确定日志中有效字节长度（从 0 开始连续解析到第一条损坏/残缺记录） */
|
||||
private findValidLogLength(log: Uint8Array): number {
|
||||
let offset = 0;
|
||||
const view = new DataView(log.buffer, log.byteOffset, log.byteLength);
|
||||
while (offset + 4 <= log.byteLength) {
|
||||
const recordLen = view.getUint32(offset, false);
|
||||
if (recordLen < 12 || offset + 4 + recordLen > log.byteLength) break;
|
||||
const raw = log.subarray(offset, offset + 4 + recordLen);
|
||||
const storedCrc = view.getUint32(offset + recordLen, false);
|
||||
if (crc32(raw.subarray(0, recordLen)) !== storedCrc) break;
|
||||
offset += 4 + recordLen;
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* KVStore Log — 追加式事务日志编解码
|
||||
* @module engine/kvstore/log
|
||||
*
|
||||
* v0.6.0: 自研 KV 引擎的原子写载体。
|
||||
* 每条日志记录 = 一个原子事务（putMany 多键写入 / deleteMany 多键删除）。
|
||||
* 单文件追加（介质 append，COW 原子）→ 崩溃时记录全有或全无。
|
||||
*
|
||||
* 记录格式（大端序）:
|
||||
* [recordLen u32] — 本条记录长度（含自身，不含 CRC）
|
||||
* [seq u32] — 日志序号（递增，恢复时与快照水位比对去重）
|
||||
* [entryCount u32] — 条目数
|
||||
* 每条 entry:
|
||||
* [op u8] — 1=PUT, 2=DELETE
|
||||
* [keyLen u32][key bytes]
|
||||
* [valueLen u32][value bytes] （DELETE 时 valueLen=0）
|
||||
* [crc u32] — 覆盖本条记录除 CRC 外全部字节的标准 CRC-32
|
||||
*/
|
||||
|
||||
import { crc32 } from '../aria/crc32';
|
||||
|
||||
/** 日志操作类型 */
|
||||
export const enum KVLogOp {
|
||||
PUT = 1,
|
||||
DELETE = 2,
|
||||
}
|
||||
|
||||
/** 解析后的日志记录 */
|
||||
export interface KVLogRecord {
|
||||
/** 日志序号 */
|
||||
seq: number;
|
||||
/** 条目列表（op, key, value） */
|
||||
entries: { op: KVLogOp; key: string; value: ArrayBuffer }[];
|
||||
/** 记录原始字节（CRC 校验用） */
|
||||
raw: Uint8Array;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编码一条日志记录。
|
||||
* @param seq 日志序号
|
||||
* @param puts key → value 写入条目
|
||||
* @param deletes 删除 key 列表
|
||||
*/
|
||||
export function encodeLogRecord(
|
||||
seq: number,
|
||||
puts: Record<string, ArrayBuffer>,
|
||||
deletes: string[] = [],
|
||||
): Uint8Array {
|
||||
const encoder = new TextEncoder();
|
||||
const entries: { op: KVLogOp; key: string; value: ArrayBuffer }[] = [];
|
||||
for (const [key, value] of Object.entries(puts)) {
|
||||
entries.push({ op: KVLogOp.PUT, key, value });
|
||||
}
|
||||
for (const key of deletes) {
|
||||
entries.push({ op: KVLogOp.DELETE, key, value: new ArrayBuffer(0) });
|
||||
}
|
||||
|
||||
// 预编码 key 字节，计算总长度
|
||||
const entryBytes: { op: KVLogOp; key: Uint8Array; value: Uint8Array }[] = [];
|
||||
let total = 4 + 4 + 4; // recordLen + seq + entryCount
|
||||
for (const e of entries) {
|
||||
const kb = encoder.encode(e.key);
|
||||
const vb = new Uint8Array(e.value);
|
||||
entryBytes.push({ op: e.op, key: kb, value: vb });
|
||||
total += 1 + 4 + kb.byteLength + 4 + vb.byteLength;
|
||||
}
|
||||
total += 4; // crc
|
||||
|
||||
const buf = new Uint8Array(total);
|
||||
const view = new DataView(buf.buffer);
|
||||
let offset = 0;
|
||||
view.setUint32(offset, total - 4, false); offset += 4; // recordLen（不含 CRC）
|
||||
view.setUint32(offset, seq, false); offset += 4;
|
||||
view.setUint32(offset, entryBytes.length, false); offset += 4;
|
||||
for (const e of entryBytes) {
|
||||
view.setUint8(offset, e.op); offset += 1;
|
||||
view.setUint32(offset, e.key.byteLength, false); offset += 4;
|
||||
buf.set(e.key, offset); offset += e.key.byteLength;
|
||||
view.setUint32(offset, e.value.byteLength, false); offset += 4;
|
||||
buf.set(e.value, offset); offset += e.value.byteLength;
|
||||
}
|
||||
const crc = crc32(buf.subarray(0, total - 4));
|
||||
view.setUint32(total - 4, crc, false);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析日志中的全部记录（顺序扫描）。
|
||||
* @param data 日志字节流
|
||||
* @param onRecord 每条有效记录回调（CRC 通过）
|
||||
* @param onCorrupt 损坏记录位置回调（返回 false 停止扫描，或继续尝试下一条）
|
||||
* @returns 有效记录数
|
||||
*/
|
||||
export function parseLogRecords(
|
||||
data: Uint8Array,
|
||||
onRecord: (record: KVLogRecord) => void,
|
||||
onCorrupt?: (offset: number) => boolean,
|
||||
): number {
|
||||
let offset = 0;
|
||||
let count = 0;
|
||||
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
while (offset + 4 <= data.byteLength) {
|
||||
const recordLen = view.getUint32(offset, false);
|
||||
if (recordLen < 12 || offset + 4 + recordLen > data.byteLength) {
|
||||
// 尾部残缺记录（最后一批写入被截断）：损坏
|
||||
if (onCorrupt) {
|
||||
if (!onCorrupt(offset)) break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
const recordStart = offset;
|
||||
const recordEnd = offset + 4 + recordLen;
|
||||
const raw = data.subarray(recordStart, recordEnd);
|
||||
|
||||
const recView = new DataView(data.buffer, data.byteOffset + recordStart, recordLen + 4);
|
||||
const storedCrc = recView.getUint32(recordLen, false);
|
||||
const computedCrc = crc32(raw.subarray(0, recordLen));
|
||||
if (storedCrc !== computedCrc) {
|
||||
if (onCorrupt) {
|
||||
if (!onCorrupt(recordStart)) break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// 解析条目
|
||||
let p = 4;
|
||||
const seq = recView.getUint32(p, false); p += 4;
|
||||
const entryCount = recView.getUint32(p, false); p += 4;
|
||||
const entries: { op: KVLogOp; key: string; value: ArrayBuffer }[] = [];
|
||||
let valid = true;
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
if (p + 1 + 4 > recordLen + 4) { valid = false; break; }
|
||||
const op = recView.getUint8(p) as KVLogOp; p += 1;
|
||||
const keyLen = recView.getUint32(p, false); p += 4;
|
||||
if (p + keyLen + 4 > recordLen + 4) { valid = false; break; }
|
||||
const key = decoder.decode(raw.subarray(p, p + keyLen)); p += keyLen;
|
||||
const valueLen = recView.getUint32(p, false); p += 4;
|
||||
if (p + valueLen > recordLen + 4) { valid = false; break; }
|
||||
const value = raw.slice(p, p + valueLen).buffer as ArrayBuffer; p += valueLen;
|
||||
entries.push({ op, key, value });
|
||||
}
|
||||
if (!valid) {
|
||||
if (onCorrupt) {
|
||||
if (!onCorrupt(recordStart)) break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
onRecord({ seq, entries, raw });
|
||||
count++;
|
||||
offset = recordEnd;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* KVStore SharedMemory Medium — 跨实例共享的内存介质
|
||||
* @module engine/kvstore/shared_memory_medium
|
||||
*
|
||||
* v0.6.0: 替代 fake-indexeddb 的测试/Node 环境介质。
|
||||
* 与 MemoryBackend 的区别：数据按库名存于全局注册表，close() 不清除
|
||||
* （模拟"磁盘持久化"语义——重新 open 同名库可读到上次写入的数据）。
|
||||
*
|
||||
* 仅用于测试与 Node 环境；浏览器使用 OPFS 介质（KVStore 默认自动选择）。
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from '../aria/store/backend';
|
||||
|
||||
/** 全局注册表：dbName → key → ArrayBuffer（跨实例共享，模拟持久化） */
|
||||
const registry = new Map<string, Map<string, ArrayBuffer>>();
|
||||
|
||||
export class SharedMemoryBackend implements IStorageBackend {
|
||||
private dbName = '';
|
||||
private store: Map<string, ArrayBuffer> | null = null;
|
||||
|
||||
/** 清空全局注册表（测试隔离用） */
|
||||
static clearRegistry(): void {
|
||||
registry.clear();
|
||||
}
|
||||
|
||||
/** 注册表中的库数量（测试诊断用） */
|
||||
static registrySize(): number {
|
||||
return registry.size;
|
||||
}
|
||||
|
||||
async open(name: string): Promise<void> {
|
||||
this.dbName = name;
|
||||
if (!registry.has(name)) {
|
||||
registry.set(name, new Map());
|
||||
}
|
||||
this.store = registry.get(name)!;
|
||||
}
|
||||
|
||||
/** close 不清除数据（持久化语义：重开同名库数据仍在） */
|
||||
async close(): Promise<void> {
|
||||
this.store = null;
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.store !== null;
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
return this.store?.get(key) ?? null;
|
||||
}
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
this.store?.set(key, data);
|
||||
}
|
||||
|
||||
async append(key: string, data: ArrayBuffer): Promise<void> {
|
||||
if (!this.store) return;
|
||||
const existing = this.store.get(key);
|
||||
if (existing) {
|
||||
const combined = new Uint8Array(existing.byteLength + data.byteLength);
|
||||
combined.set(new Uint8Array(existing), 0);
|
||||
combined.set(new Uint8Array(data), existing.byteLength);
|
||||
this.store.set(key, combined.buffer as ArrayBuffer);
|
||||
} else {
|
||||
this.store.set(key, data);
|
||||
}
|
||||
}
|
||||
|
||||
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
|
||||
if (!this.store) return;
|
||||
// 同步批量写入 = 原子（JS 单线程，无中间 await 点）
|
||||
for (const [key, data] of Object.entries(entries)) {
|
||||
this.store.set(key, data);
|
||||
}
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
this.store?.delete(key);
|
||||
}
|
||||
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
if (!this.store) return;
|
||||
for (const key of keys) {
|
||||
this.store.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
return this.store ? Array.from(this.store.keys()) : [];
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
return this.store?.has(key) ?? false;
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
this.store?.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* KVStore Snapshot — 快照序列化/反序列化
|
||||
* @module engine/kvstore/snapshot
|
||||
*
|
||||
* v0.6.0: checkpoint 时把全部 key-value 序列化为快照文件（COW 原子写），
|
||||
* 快照内嵌"日志水位 seq"（快照包含的最后一条日志序号），恢复时只重放 seq > 水位 的记录。
|
||||
*
|
||||
* 格式（大端序）:
|
||||
* [magic u32] — 0x4B56534E ("KVSN")
|
||||
* [seq u32] — 日志水位（快照包含的数据对应的日志序号）
|
||||
* [entryCount u32]
|
||||
* 每条: [keyLen u32][key bytes][valueLen u32][value bytes]
|
||||
* [crc u32] — 覆盖除 CRC 外全部字节的标准 CRC-32
|
||||
*/
|
||||
|
||||
import { crc32 } from '../aria/crc32';
|
||||
|
||||
const SNAPSHOT_MAGIC = 0x4b56534e; // "KVSN"
|
||||
|
||||
/** 快照内容 */
|
||||
export interface KVSsnapshot {
|
||||
/** 日志水位 */
|
||||
seq: number;
|
||||
/** key → value */
|
||||
entries: Map<string, ArrayBuffer>;
|
||||
}
|
||||
|
||||
/** 序列化快照 */
|
||||
export function encodeSnapshot(seq: number, entries: Map<string, ArrayBuffer>): Uint8Array {
|
||||
const encoder = new TextEncoder();
|
||||
const keys = Array.from(entries.keys());
|
||||
|
||||
// 预编码
|
||||
const encoded: { key: Uint8Array; value: Uint8Array }[] = [];
|
||||
let total = 4 + 4 + 4; // magic + seq + entryCount
|
||||
for (const key of keys) {
|
||||
const kb = encoder.encode(key);
|
||||
const vb = new Uint8Array(entries.get(key)!);
|
||||
encoded.push({ key: kb, value: vb });
|
||||
total += 4 + kb.byteLength + 4 + vb.byteLength;
|
||||
}
|
||||
total += 4; // crc
|
||||
|
||||
const buf = new Uint8Array(total);
|
||||
const view = new DataView(buf.buffer);
|
||||
let offset = 0;
|
||||
view.setUint32(offset, SNAPSHOT_MAGIC, false); offset += 4;
|
||||
view.setUint32(offset, seq, false); offset += 4;
|
||||
view.setUint32(offset, encoded.length, false); offset += 4;
|
||||
for (const e of encoded) {
|
||||
view.setUint32(offset, e.key.byteLength, false); offset += 4;
|
||||
buf.set(e.key, offset); offset += e.key.byteLength;
|
||||
view.setUint32(offset, e.value.byteLength, false); offset += 4;
|
||||
buf.set(e.value, offset); offset += e.value.byteLength;
|
||||
}
|
||||
const crc = crc32(buf.subarray(0, total - 4));
|
||||
view.setUint32(total - 4, crc, false);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析快照。
|
||||
* @returns 快照内容；损坏（magic 错误/CRC 失败/越界）返回 null
|
||||
*/
|
||||
export function decodeSnapshot(data: Uint8Array): KVSsnapshot | null {
|
||||
if (data.byteLength < 16) return null;
|
||||
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
if (view.getUint32(0, false) !== SNAPSHOT_MAGIC) return null;
|
||||
|
||||
const storedCrc = view.getUint32(data.byteLength - 4, false);
|
||||
const computedCrc = crc32(data.subarray(0, data.byteLength - 4));
|
||||
if (storedCrc !== computedCrc) return null;
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
const entries = new Map<string, ArrayBuffer>();
|
||||
let p = 4;
|
||||
const seq = view.getUint32(p, false); p += 4;
|
||||
const entryCount = view.getUint32(p, false); p += 4;
|
||||
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
if (p + 4 > data.byteLength - 4) return null;
|
||||
const keyLen = view.getUint32(p, false); p += 4;
|
||||
if (p + keyLen + 4 > data.byteLength - 4) return null;
|
||||
const key = decoder.decode(data.subarray(p, p + keyLen)); p += keyLen;
|
||||
const valueLen = view.getUint32(p, false); p += 4;
|
||||
if (p + valueLen > data.byteLength - 4) return null;
|
||||
const value = data.slice(p, p + valueLen).buffer as ArrayBuffer; p += valueLen;
|
||||
entries.set(key, value);
|
||||
}
|
||||
|
||||
return { seq, entries };
|
||||
}
|
||||
Reference in New Issue
Block a user