release: v0.6.0 — 完全移除 IndexedDB，自研 KVStore 事务存储引擎（多key原子写/快照日志恢复/CRC自愈）+ KVStoreEngine + 旧库迁移工具 + 10万级压力验证 + 崩溃注入e2e
This commit is contained in:
thzxx
@@ -0,0 +1,492 @@
|
||||
/**
|
||||
* KVStoreEngine — 基于自研 KVStore 的磁盘存储引擎（替代 IndexedDBEngine / OPFSEngine）
|
||||
* @module engine/kvstore_engine
|
||||
*
|
||||
* v0.6.0: 完全移除 IndexedDB 后的 disk 模式引擎。
|
||||
*
|
||||
* 架构：MemoryEngine（内存热路径 + 事务快照）+ KVStore（持久化 + 原子写）
|
||||
* - 读：始终走内存（写路径同步落盘，重启从 KVStore 恢复）
|
||||
* - 写：内存先行 + KVStore 增量持久化（insert 增量 putMany；update/delete 受影响行重写；
|
||||
* 主键变更/级联场景整表 diff；全部原子）
|
||||
* - 事务：内存快照 + commit 时受影响表原子 flush（putMany 单记录 = 真原子，
|
||||
* 此前 IndexedDBEngine 依赖 IDB 事务，现在完全自研）
|
||||
*
|
||||
* 数据布局（KVStore keys）:
|
||||
* `__schema` — JSON { tableName: TableSchema }
|
||||
* `__meta:{key}` — 库内元数据（迁移版本等）
|
||||
* `t:{table}:{pk}` — 行数据（JSON）
|
||||
*/
|
||||
|
||||
import type { IStorageEngine } from './interface';
|
||||
import type { QueryPlan, TableSchema } from '../constants';
|
||||
import { DatabaseError } from '../constants';
|
||||
import { MemoryEngine } from './memory';
|
||||
import { KVStore } from './kvstore/index';
|
||||
import { SharedMemoryBackend } from './kvstore/shared_memory_medium';
|
||||
import type { IStorageBackend } from './aria/store/backend';
|
||||
|
||||
const SCHEMA_KEY = '__schema';
|
||||
const ROW_PREFIX = 't:';
|
||||
|
||||
const enc = (s: string) => new TextEncoder().encode(s).buffer as ArrayBuffer;
|
||||
const dec = (b: ArrayBuffer) => new TextDecoder().decode(b);
|
||||
|
||||
export class KVStoreEngine implements IStorageEngine {
|
||||
readonly name = 'kv';
|
||||
|
||||
private kv!: KVStore;
|
||||
private memory: MemoryEngine = new MemoryEngine();
|
||||
private dbName = '';
|
||||
private version = 1;
|
||||
private opened = false;
|
||||
|
||||
/** 活跃事务标记 */
|
||||
private txActive = false;
|
||||
/** 事务中写过的表（commit 时只 flush 这些表） */
|
||||
private txDirtyTables: Set<string> = new Set();
|
||||
|
||||
constructor(medium?: IStorageBackend, checkpointThreshold?: number) {
|
||||
this.kv = new KVStore(medium, checkpointThreshold);
|
||||
}
|
||||
|
||||
// ---- 行 key 编解码 ----
|
||||
|
||||
private rowKey(table: string, pk: string): string {
|
||||
return `${ROW_PREFIX}${table}:${pk}`;
|
||||
}
|
||||
|
||||
private rowPrefix(table: string): string {
|
||||
return `${ROW_PREFIX}${table}:`;
|
||||
}
|
||||
|
||||
// ---- 生命周期 ----
|
||||
|
||||
async open(dbName: string, version: number): Promise<void> {
|
||||
if (this.opened) return;
|
||||
this.dbName = dbName;
|
||||
this.version = version;
|
||||
await this.kv.open(dbName);
|
||||
await this.memory.open(dbName, version);
|
||||
|
||||
// 恢复 schema
|
||||
const schemaRaw = await this.kv.get(SCHEMA_KEY);
|
||||
if (schemaRaw) {
|
||||
try {
|
||||
const schemas = JSON.parse(dec(schemaRaw)) as Record<string, TableSchema>;
|
||||
for (const schema of Object.values(schemas)) {
|
||||
await this.memory.createTable(schema);
|
||||
}
|
||||
} catch {
|
||||
throw new DatabaseError('Corrupted schema in KVStore', 'KV_SCHEMA_ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复行数据 + 重建索引
|
||||
const all = await this.kv.getAll();
|
||||
for (const [key, value] of all) {
|
||||
if (!key.startsWith(ROW_PREFIX)) continue;
|
||||
const sep = key.indexOf(':', ROW_PREFIX.length);
|
||||
if (sep < 0) continue;
|
||||
const table = key.slice(ROW_PREFIX.length, sep);
|
||||
if (!(await this.memory.hasTable(table))) continue;
|
||||
try {
|
||||
const row = JSON.parse(dec(value));
|
||||
await this.memory.insert(table, [row]);
|
||||
} catch {
|
||||
// 单行损坏跳过（repair 可清理）
|
||||
}
|
||||
}
|
||||
// 重建二级索引（schema 标记的索引列）
|
||||
const tables = await this.memory.getTableNames();
|
||||
for (const table of tables) {
|
||||
const schema = await this.memory.getTableSchema(table);
|
||||
if (!schema) continue;
|
||||
for (const [col, colDef] of Object.entries(schema.columns)) {
|
||||
if (colDef.index || colDef.unique) {
|
||||
await this.memory.createIndex(table, col, colDef.unique);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.opened = true;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (!this.opened) return;
|
||||
// 活跃事务先回滚
|
||||
if (this.txActive) {
|
||||
try { await this.rollbackTransaction(); } catch { /* ignore */ }
|
||||
}
|
||||
await this.kv.close();
|
||||
await this.memory.close();
|
||||
this.opened = false;
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.opened;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.6.0: 从 KVStore 重新加载全部数据到内存（多标签页同步重载用）。
|
||||
* Hybrid 引擎的 reloadMemoryFromDisk 依赖磁盘引擎"读穿透"，
|
||||
* KVStoreEngine 读内存 → 提供 reload 重新加载磁盘最新数据。
|
||||
*/
|
||||
async reload(): Promise<void> {
|
||||
if (!this.opened) return;
|
||||
// 1. KVStore 重新从介质加载（外部写入可见）
|
||||
await this.kv.reload();
|
||||
// 2. 内存缓存重载
|
||||
await this.memory.close();
|
||||
await this.memory.open(this.dbName, this.version);
|
||||
this.opened = false;
|
||||
await this.open(this.dbName, this.version);
|
||||
}
|
||||
|
||||
/** v0.4.2-fix: 自愈 — 校验 KVStore 日志/快照完整性并重建内存 */
|
||||
async repair(): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.kv.repair();
|
||||
await this.memory.close();
|
||||
await this.memory.open(this.dbName, this.version);
|
||||
// 重新恢复（复用 open 的恢复逻辑）
|
||||
this.opened = false;
|
||||
await this.open(this.dbName, this.version);
|
||||
}
|
||||
|
||||
async clearAll(): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.kv.clear();
|
||||
await this.memory.clearAll();
|
||||
}
|
||||
|
||||
async getMeta(key: string): Promise<string | null> {
|
||||
const raw = await this.kv.get(`__meta:${key}`);
|
||||
return raw ? dec(raw) : null;
|
||||
}
|
||||
|
||||
async setMeta(key: string, value: string): Promise<void> {
|
||||
await this.kv.put(`__meta:${key}`, enc(value));
|
||||
}
|
||||
|
||||
// ---- 表管理 ----
|
||||
|
||||
async createTable(schema: TableSchema): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.createTable(schema);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(schema.name);
|
||||
return;
|
||||
}
|
||||
await this.persistSchema();
|
||||
}
|
||||
|
||||
async dropTable(tableName: string): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.dropTable(tableName);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return;
|
||||
}
|
||||
await this.persistSchema();
|
||||
// 删除该表全部行（KV 中残留清理）
|
||||
await this.flushTable(tableName);
|
||||
}
|
||||
|
||||
async hasTable(tableName: string): Promise<boolean> {
|
||||
return this.memory.hasTable(tableName);
|
||||
}
|
||||
|
||||
async getTableNames(): Promise<string[]> {
|
||||
return this.memory.getTableNames();
|
||||
}
|
||||
|
||||
async getTableSchema(tableName: string): Promise<TableSchema | null> {
|
||||
return this.memory.getTableSchema(tableName);
|
||||
}
|
||||
|
||||
async alterTable(
|
||||
tableName: string,
|
||||
action: 'ADD' | 'DROP',
|
||||
column: import('../constants').ColumnDef & { name: string },
|
||||
): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.alterTable(tableName, action, column);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return;
|
||||
}
|
||||
await this.persistSchema();
|
||||
if (action === 'DROP') {
|
||||
// 重写存储行（移除该列）
|
||||
await this.flushTable(tableName);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- CRUD ----
|
||||
|
||||
async insert(tableName: string, rows: Record<string, unknown>[]): Promise<string[]> {
|
||||
this.ensureOpen();
|
||||
const pks = await this.memory.insert(tableName, rows);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return pks;
|
||||
}
|
||||
// 增量持久化（原子 putMany）
|
||||
const schema = await this.memory.getTableSchema(tableName);
|
||||
if (!schema) throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
|
||||
const pkCol = this.getPK(schema);
|
||||
const puts: Record<string, ArrayBuffer> = {};
|
||||
rows.forEach((row, i) => {
|
||||
puts[this.rowKey(tableName, String(pks[i] ?? row[pkCol]))] = enc(JSON.stringify(row));
|
||||
});
|
||||
await this.kv.putMany(puts);
|
||||
return pks;
|
||||
}
|
||||
|
||||
async find(tableName: string, query: QueryPlan): Promise<Record<string, unknown>[]> {
|
||||
this.ensureOpen();
|
||||
return this.memory.find(tableName, query);
|
||||
}
|
||||
|
||||
async findStream(tableName: string, query: QueryPlan, onRow: (row: Record<string, unknown>) => void): Promise<number> {
|
||||
this.ensureOpen();
|
||||
return this.memory.findStream(tableName, query, onRow);
|
||||
}
|
||||
|
||||
async update(
|
||||
tableName: string,
|
||||
query: QueryPlan,
|
||||
updates: Record<string, unknown>,
|
||||
): Promise<number> {
|
||||
this.ensureOpen();
|
||||
const schema = await this.memory.getTableSchema(tableName);
|
||||
if (!schema) throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
|
||||
const pkCol = this.getPK(schema);
|
||||
const pkChanged = pkCol in updates;
|
||||
|
||||
// 收集受影响旧主键（内存匹配）
|
||||
const affected = pkChanged ? [] : await this.collectMatchingPks(tableName, query);
|
||||
const count = await this.memory.update(tableName, query, updates);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return count;
|
||||
}
|
||||
|
||||
if (pkChanged) {
|
||||
// 主键变更：相关表整表 diff（罕见操作，可靠性优先）
|
||||
for (const t of await this.affectedTables(tableName)) {
|
||||
await this.flushTable(t);
|
||||
}
|
||||
} else {
|
||||
// 增量重写受影响行
|
||||
const puts: Record<string, ArrayBuffer> = {};
|
||||
const deletes: string[] = [];
|
||||
for (const pk of affected) {
|
||||
const row = await this.memory.find(tableName, { table: tableName, where: { [pkCol]: pk } });
|
||||
if (row.length > 0) {
|
||||
puts[this.rowKey(tableName, pk)] = enc(JSON.stringify(row[0]));
|
||||
} else {
|
||||
deletes.push(this.rowKey(tableName, pk));
|
||||
}
|
||||
}
|
||||
if (Object.keys(puts).length > 0) await this.kv.putMany(puts);
|
||||
if (deletes.length > 0) await this.kv.deleteMany(deletes);
|
||||
// 级联影响表（SET NULL/CASCADE 外键）整表 diff
|
||||
for (const t of await this.affectedTables(tableName)) {
|
||||
if (t !== tableName) await this.flushTable(t);
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
async delete(tableName: string, query: QueryPlan): Promise<number> {
|
||||
this.ensureOpen();
|
||||
// 收集受影响主键（内存匹配）
|
||||
const pks = await this.collectMatchingPks(tableName, query);
|
||||
const count = await this.memory.delete(tableName, query);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return count;
|
||||
}
|
||||
const deletes = pks.map((pk) => this.rowKey(tableName, pk));
|
||||
if (deletes.length > 0) await this.kv.deleteMany(deletes);
|
||||
// 级联影响表整表 diff
|
||||
for (const t of await this.affectedTables(tableName)) {
|
||||
if (t !== tableName) await this.flushTable(t);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
async count(tableName: string, query?: QueryPlan): Promise<number> {
|
||||
this.ensureOpen();
|
||||
return this.memory.count(tableName, query);
|
||||
}
|
||||
|
||||
async clear(tableName: string): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.clear(tableName);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return;
|
||||
}
|
||||
await this.flushTable(tableName);
|
||||
}
|
||||
|
||||
// ---- 动态索引 ----
|
||||
|
||||
async createIndex(tableName: string, column: string, unique?: boolean): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.createIndex(tableName, column, unique);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return;
|
||||
}
|
||||
await this.persistSchema();
|
||||
}
|
||||
|
||||
async dropIndex(tableName: string, column: string, indexName?: string): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.dropIndex(tableName, column, indexName);
|
||||
if (this.txActive) {
|
||||
this.txDirtyTables.add(tableName);
|
||||
return;
|
||||
}
|
||||
await this.persistSchema();
|
||||
}
|
||||
|
||||
// ---- 事务（原子 flush） ----
|
||||
|
||||
async beginTransaction(): Promise<void> {
|
||||
this.ensureOpen();
|
||||
await this.memory.beginTransaction();
|
||||
this.txActive = true;
|
||||
this.txDirtyTables = new Set();
|
||||
}
|
||||
|
||||
async commitTransaction(): Promise<void> {
|
||||
this.ensureOpen();
|
||||
if (!this.txActive) throw new DatabaseError('No active transaction', 'TX_NONE');
|
||||
// 先持久化（原子），再提交内存快照（失败可回滚）
|
||||
for (const table of this.txDirtyTables) {
|
||||
if (await this.memory.hasTable(table)) {
|
||||
await this.flushTable(table);
|
||||
} else {
|
||||
// 事务内 drop 的表：清理 KV 残留行
|
||||
const all = await this.kv.getAll();
|
||||
const prefix = this.rowPrefix(table);
|
||||
const deletes = all.filter(([key]) => key.startsWith(prefix)).map(([key]) => key);
|
||||
if (deletes.length > 0) await this.kv.deleteMany(deletes);
|
||||
}
|
||||
}
|
||||
// v0.6.0-fix: 事务内 DDL（create/drop/alter）的 schema 一并持久化
|
||||
await this.persistSchema();
|
||||
await this.kv.checkpoint();
|
||||
await this.memory.commitTransaction();
|
||||
this.txActive = false;
|
||||
this.txDirtyTables = new Set();
|
||||
}
|
||||
|
||||
async rollbackTransaction(): Promise<void> {
|
||||
this.ensureOpen();
|
||||
if (!this.txActive) throw new DatabaseError('No active transaction', 'TX_NONE');
|
||||
await this.memory.rollbackTransaction();
|
||||
this.txActive = false;
|
||||
this.txDirtyTables = new Set();
|
||||
}
|
||||
|
||||
// ---- 内部 ----
|
||||
|
||||
private ensureOpen(): void {
|
||||
if (!this.opened) throw new DatabaseError('Database not opened', 'DB_NOT_OPEN');
|
||||
}
|
||||
|
||||
private getPK(schema: TableSchema): string {
|
||||
for (const [name, col] of Object.entries(schema.columns)) {
|
||||
if (col.primaryKey) return name;
|
||||
}
|
||||
return Object.keys(schema.columns)[0];
|
||||
}
|
||||
|
||||
/** 收集匹配查询的内存行主键（持久化差异计算用） */
|
||||
private async collectMatchingPks(tableName: string, query: QueryPlan): Promise<string[]> {
|
||||
const schema = await this.memory.getTableSchema(tableName);
|
||||
if (!schema) throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
|
||||
const pkCol = this.getPK(schema);
|
||||
const rows = await this.memory.find(tableName, query);
|
||||
return rows.map((r) => String(r[pkCol]));
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算外键级联影响的表集合（传递闭包：A 被 B 引用，B 被 C 引用 → {A, B, C}）。
|
||||
* 级联操作（delete/update 主键）需要把这些表一并重写持久化。
|
||||
*/
|
||||
private async affectedTables(tableName: string): Promise<Set<string>> {
|
||||
const set = new Set<string>([tableName]);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const table of await this.memory.getTableNames()) {
|
||||
if (set.has(table)) continue;
|
||||
const schema = await this.memory.getTableSchema(table);
|
||||
if (!schema) continue;
|
||||
for (const col of Object.values(schema.columns)) {
|
||||
if (col.references) {
|
||||
const ref = col.references.split('.')[0];
|
||||
if (set.has(ref)) {
|
||||
set.add(table);
|
||||
changed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
/** 持久化 schema（全部表） */
|
||||
private async persistSchema(): Promise<void> {
|
||||
const schemas: Record<string, TableSchema> = {};
|
||||
for (const table of await this.memory.getTableNames()) {
|
||||
const schema = await this.memory.getTableSchema(table);
|
||||
if (schema) schemas[table] = schema;
|
||||
}
|
||||
await this.kv.put(SCHEMA_KEY, enc(JSON.stringify(schemas)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 整表 diff 持久化：内存行全部 put + KV 残留行删除（原子 putMany + deleteMany）。
|
||||
* 用于主键变更 / 级联 / dropTable / clear / alterTable DROP / 事务 commit。
|
||||
*/
|
||||
private async flushTable(tableName: string): Promise<void> {
|
||||
const prefix = this.rowPrefix(tableName);
|
||||
// 表已删除：仅清理 KV 残留行
|
||||
const schema = await this.memory.getTableSchema(tableName);
|
||||
if (!schema) {
|
||||
const all = await this.kv.getAll();
|
||||
const deletes = all.filter(([key]) => key.startsWith(prefix)).map(([key]) => key);
|
||||
if (deletes.length > 0) await this.kv.deleteMany(deletes);
|
||||
return;
|
||||
}
|
||||
const pkCol = this.getPK(schema);
|
||||
const rows = await this.memory.find(tableName, { table: tableName });
|
||||
|
||||
const puts: Record<string, ArrayBuffer> = {};
|
||||
const current = new Set<string>();
|
||||
for (const row of rows) {
|
||||
const key = this.rowKey(tableName, String(row[pkCol]));
|
||||
current.add(key);
|
||||
puts[key] = enc(JSON.stringify(row));
|
||||
}
|
||||
// KV 残留行（内存中已不存在）删除
|
||||
const all = await this.kv.getAll();
|
||||
const deletes: string[] = [];
|
||||
for (const [key] of all) {
|
||||
if (key.startsWith(prefix) && !current.has(key)) {
|
||||
deletes.push(key);
|
||||
}
|
||||
}
|
||||
if (Object.keys(puts).length > 0) await this.kv.putMany(puts);
|
||||
if (deletes.length > 0) await this.kv.deleteMany(deletes);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user