import { cloneRow } from '../interface'; /** * AriaEngine — 自研页面式存储引擎主类 * @module engine/aria/index * * v0.4.1: 外键级联 + ALTER TABLE 重写 + clearAll 重置 + 崩溃恢复加固 */ import type { IStorageEngine } from '../interface'; import type { QueryPlan, TableSchema, ColumnDef, WhereCondition } from '../../constants'; import { DatabaseError } from '../../constants'; import { matchWhere, applyOrderBy, projectColumns, containsUnresolvedSubqueries } from '../../query/where-matcher'; import { checkFieldType, stripUndefinedUpdates } from '../../table/schema'; import type { AriaEngineConfig, SSTableMeta } from './types'; import { DEFAULT_ARIA_CONFIG } from './types'; import { LSM } from './index/lsm'; import type { SSTableStore } from './index/lsm'; import { WAL } from './wal/log'; import { SegmentedWALStore } from './wal/segmented_store'; import { DatabaseLock } from './locks'; import { WALRecordType, type WALRecord } from './types'; import { CheckpointManager } from './wal/checkpoint'; import { MemoryBackend, type IStorageBackend } from './store/backend'; import { KVStoreBackend } from './store/kvstore_backend'; import { OPFSBackend } from './store/opfs_backend'; import { EncryptedBackend } from './store/encrypted_backend'; import { PageSSTableStore } from './store/page_sstable_store'; import { FileManager } from './store/file_manager'; import { MVCCManager } from './transaction/mvcc'; import { BufferPool } from './buffer/pool'; import { compressLZ4, decompressLZ4 } from './compression/lz4'; import type { WALRecord as _WALRecord } from './types'; // --------------------------------------------------------------------------- // AriaEngine // --------------------------------------------------------------------------- export class AriaEngine implements IStorageEngine { readonly name = 'aria'; private config!: Required> & Pick; private lsm!: LSM; // 主键索引 LSM private wal!: WAL; private checkpointManager!: CheckpointManager; private backend!: IStorageBackend; private opened = false; private dbName = ''; // v0.4.5: 页面化物理存储(FileManager + BufferPool 提升为引擎字段,close/repair 时落盘/清理) private fileManager!: FileManager; private bufferPool!: BufferPool; // v0.4.5: 多标签页独占锁(Web Locks API,OPFS 等无事务后端防并发写) private dbLock: DatabaseLock | null = null; // 表结构 private schemas: Map = new Map(); private tablePKs: Map = new Map(); private opCounter = 0; // 二级索引:table.colKey → LSM private secondaryIndexes: Map = new Map(); /** * v0.7.4: 由 CREATE UNIQUE INDEX 添加的 unique 列(table:col)。 * 与建表 UNIQUE 约束区分:DROP INDEX 只允许解除索引来源的 unique, * 建表约束需重建表(对齐 SQLite 语义,此前静默解除且重启后永久消失)。 * 注:重启后无法区分历史来源,schema 中的 unique 一律按建表约束保护(保守)。 */ private uniqueIndexCols: Set = new Set(); // MVCC 事务 private mvcc: MVCCManager = new MVCCManager(); private currentTxnId: number | null = null; private txnSnapshot: Map> | null = null; private gcCounter = 0; constructor(config: AriaEngineConfig = {}) { this.config = { ...DEFAULT_ARIA_CONFIG, ...config }; } // ======================================================================= // 生命周期 // ======================================================================= async open(dbName: string, _version: number): Promise { if (this.opened) return; // v0.4.2-fix: 引擎内部错误统一包装为 DatabaseError(ARIA_OPEN_ERROR), // 应用层可拿到 code 分类处理,不再抛出原生 RangeError/TypeError try { await this.openInternal(dbName); } catch (error) { // 打开失败:释放已获取的锁(避免锁泄漏阻塞其他标签页) if (this.dbLock) { try { await this.dbLock.release(); } catch { /* ignore */ } this.dbLock = null; } if (error instanceof DatabaseError) throw error; throw new DatabaseError( `Failed to open AriaEngine database "${dbName}"`, 'ARIA_OPEN_ERROR', error, ); } } /** open 内部实现(错误包装在 open 外层) */ private async openInternal(dbName: string): Promise { this.dbName = dbName; // v0.4.5: 多标签页独占锁(Web Locks)— 不支持的环境降级为无锁(文档注明) const lock = new DatabaseLock(); this.dbLock = lock; const lockAcquired = await lock.acquire(dbName); if (!lockAcquired) { // eslint-disable-next-line no-console console.warn( `[AriaEngine] Web Locks API unavailable: no multi-tab protection for "${dbName}" ` + '(open the same database in multiple tabs may corrupt data)', ); } // 1. 存储后端(可选全库加密包装) let baseBackend: IStorageBackend; if (this.config.storageBackend === 'opfs') { baseBackend = new OPFSBackend(); } else if (this.config.storageBackend === 'kv') { // v0.6.1: 自研 KVStore 后端(aria 完全跑在自研存储栈上,不依赖浏览器 OPFS) baseBackend = new KVStoreBackend(); } else { baseBackend = new MemoryBackend(); } await baseBackend.open(dbName); // v0.4.5: encryption 配置 → 透明加密封装(密码错误/数据损坏在 open 或首次读取时暴露) if (this.config.encryption?.password) { this.backend = new EncryptedBackend(baseBackend, this.config.encryption.password); } else { this.backend = baseBackend; // 反向检测:库中存在密钥元数据但未提供密码 → 拒绝打开(避免密文被当明文解析成空库) if (await baseBackend.exists('__aria_keymeta')) { await baseBackend.close(); throw new DatabaseError( 'Database is encrypted: provide encryption.password to open it', 'ARIA_ENCRYPT_REQUIRED', ); } } await this.backend.open(dbName); // 2a. FileManager (PageIO 实现) + Buffer Pool this.fileManager = new FileManager(this.backend); await this.fileManager.init(dbName); this.bufferPool = new BufferPool(this.fileManager, this.config.bufferPoolPages); // 2. 构建 SSTableStore const sstableStore = this.createSSTableStore('main'); // 3. 初始化主 LSM(PK 索引) this.lsm = new LSM({ memtableSizeThreshold: this.config.memtableSizeThreshold, levelSizeMultiplier: this.config.levelSizeMultiplier, blockSize: this.config.pageSize, bloomBitsPerKey: this.config.bloomFilterBitsPerKey, // SSTable 缓存上限 = BufferPool 页数 × 页面大小(默认 256 页 ≈ 1MB 可控内存) cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, sstableStore, }); // 4. 初始化 WAL // v0.4.5: 分片式 WAL 存储(__wal_%06d.bin),序号内嵌记录字节流无需 count 键, // append 单文件原子写;空洞检测截断;兼容旧格式 __wal_N + __wal_count this.wal = new WAL( new SegmentedWALStore(this.backend), this.config.walEnabled, this.config.walSyncMode, ); // 5. 恢复 Schema await this.loadSchemas(); // v0.4.2-fix: 为 schema 中带 index/unique 标记的列重建二级索引 LSM。 // 此前重开只恢复 schema 不恢复索引 LSM → 索引查询静默回退全表、 // createIndex 因 colDef 已有标记直接 return → 索引永久缺失。 // 索引数据已持久化在独立命名空间(sst_idx_* / meta),init() 直接加载。 for (const [tableName, schema] of this.schemas) { const pkCol = this.tablePKs.get(tableName)!; for (const [colName, colDef] of Object.entries(schema.columns)) { if ((colDef.index || colDef.unique) && colName !== pkCol) { const idxKey = `${tableName}:idx:${colName}`; if (!this.secondaryIndexes.has(idxKey)) { const idxLsm = new LSM({ memtableSizeThreshold: this.config.memtableSizeThreshold, levelSizeMultiplier: this.config.levelSizeMultiplier, blockSize: this.config.pageSize, bloomBitsPerKey: this.config.bloomFilterBitsPerKey, cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, sstableStore: this.createSSTableStore(`idx_${tableName}_${colName}`), }); await idxLsm.init(); this.secondaryIndexes.set(idxKey, idxLsm); } } } } // 6. 初始化 LSM(加载 SSTable 元数据) await this.lsm.init(); // 7. WAL 恢复(两阶段:先扫描事务边界,仅回放已提交事务) const committedTxns = new Set(); const allRecords: WALRecord[] = []; await this.wal.recover((r) => allRecords.push(r)); // 第一遍:确定已提交事务 for (const r of allRecords) { if (r.type === WALRecordType.COMMIT) committedTxns.add(r.txnId); if (r.type === WALRecordType.ROLLBACK) committedTxns.delete(r.txnId); } // v0.8.0 根治:确定每个事务的**回放起始下标**。 // // `ROLLBACK TO ` 只回滚内存快照,日志里仍留有 savepoint 之前 // 写入的记录;COMMIT 又把整个 txnId 标记为已提交 —— 于是那些被回滚掉的写入 // 在重启时被重新应用,**已回滚的行复活**(实测:实时只剩 a,崩溃重开变成 a+b)。 // // 现在 savepoint 回滚会写入 SAVEPOINT_ROLLBACK 记录;恢复时该事务只应用 // **最后一条** SAVEPOINT_ROLLBACK 之后的记录 —— 等价于"回到该保存点", // 与实时态严格一致(这样也就不需要 undo 崩溃前已落盘的旧值)。 // 每个事务:{ 保留起点, 丢弃终点 } —— 区间 [start, end) 保留。 // // 关键:`replayFromIndex` 是**事务内**的记录序号(写入方按事务计数), // 因此这里必须用"该事务的第几条记录"来比较,不能直接拿全局下标 —— // 全局下标里还混着 txnId=0 的非事务记录(CREATE_TABLE 等)以及其它事务。 const globalIndexToTxnIndex = new Map(); const txnRecordCount = new Map(); for (let i = 0; i < allRecords.length; i++) { const r = allRecords[i]; if (r.txnId === 0) continue; const n = txnRecordCount.get(r.txnId) ?? 0; globalIndexToTxnIndex.set(i, n); txnRecordCount.set(r.txnId, n + 1); } // 边界语义(必须显式定义,否则差一错误就在这里): // 写入侧记录的 `replayFromIndex = N` 表示"保存点建立时,本事务已成功追加了 N 条记录", // 即事务的第 0..N-1 条记录必须保留(BEGIN 是第 0 条)。 // 因此恢复侧应保留的**事务内下标区间**是 [0, N),丢弃 [N, 标记位置)。 // 等价地:只丢弃"事务内下标 >= N"且"在最后一个标记之前"的记录。 const txnReplayWindow = new Map(); for (let i = 0; i < allRecords.length; i++) { const r = allRecords[i]; if (r.type !== WALRecordType.SAVEPOINT_ROLLBACK) continue; const declared = (r.data as { replayFromIndex?: number } | undefined)?.replayFromIndex; const keepUpTo = typeof declared === 'number' ? declared : 0; const txnIdx = globalIndexToTxnIndex.get(i) ?? 0; const prev = txnReplayWindow.get(r.txnId); // 多个保存点回滚:保留上界取**最早**的(回到最早的保存点), // 丢弃起点取**最后一个**标记的事务内下标。 txnReplayWindow.set(r.txnId, { keepUpTo: prev ? Math.min(prev.keepUpTo, keepUpTo) : keepUpTo, dropFrom: txnIdx, }); } // 第二遍:仅应用 txnId==0(非事务)或已提交事务的数据 for (let i = 0; i < allRecords.length; i++) { const r = allRecords[i]; if (r.txnId === 0 || committedTxns.has(r.txnId)) { if (r.type === WALRecordType.SAVEPOINT_ROLLBACK) continue; // 标记记录,无数据 // 事务内落在 [start, end) 之外的记录:属于被 savepoint 回滚掉的部分,丢弃 const win = txnReplayWindow.get(r.txnId); if (win !== undefined) { const txnIdx = globalIndexToTxnIndex.get(i) ?? 0; // 保留 [0, keepUpTo);丢弃 [keepUpTo, dropFrom);标记之后的记录照常保留 if (txnIdx >= win.keepUpTo && txnIdx < win.dropFrom) continue; } if (r.type === WALRecordType.DROP_TABLE) { // v0.3.3: DROP_TABLE 回放(异步:需预加载 SSTable 后清除残留数据) await this.applyDropTableRecovery(r.tableName); } else { this.applyWALRecord(r); } } } // v0.3.3: 恢复完成后将回放数据落盘并截断 WAL, // 避免每次重启重复回放 + WAL 无限膨胀 if (allRecords.length > 0) { await this.lsm.flush(); await this.wal.checkpoint(); // v0.4.2-fix: WAL 回放只更新主 LSM,二级索引 LSM 未同步 → // 崩溃前最后一批写入的索引缺失,重开时索引查询丢行。 // 恢复后全量重建所有表的二级索引(幂等)。 for (const tableName of this.schemas.keys()) { await this.reindexTableInternal(tableName); } } // 8. Checkpoint Manager(接入 WAL 大小阈值) // v0.4.2-fix: 事务活跃时 checkpoint 不得截断 WAL — // 否则 BEGIN/INSERT 记录被截断,COMMIT 后崩溃恢复丢失整个事务数据 this.checkpointManager = new CheckpointManager( this.lsm, { checkpoint: async () => { if (this.currentTxnId) return; await this.wal.checkpoint(); }, flush: async () => { if (this.currentTxnId) return; await this.wal.flush(); }, getBufferedBytes: () => this.wal.getBufferedBytes(), getBufferedCount: () => this.wal.getBufferedCount(), } as unknown as WAL, { flushAll: async () => { // v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM — // 此前只 flush 主 LSM,checkpoint 截断 WAL 后崩溃时索引 memtable 未落盘、 // WAL 为空跳过重建 → 二级索引静默丢失最后一批条目(生产数据一致性问题) await this.lsm.flush(); for (const idxLsm of this.secondaryIndexes.values()) { await idxLsm.flush(); } }, } as any, this.config.checkpointInterval, this.config.walSizeThreshold, ); this.opened = true; } async close(): Promise { if (!this.opened) return; await this.persistSchemas(); await this.lsm.flush(); // v0.4.2-fix: 同步落盘全部二级索引 LSM — 此前只 flush 主 LSM, // 优雅关闭后索引 memtable 未落盘 → 重开索引为空 → 索引查询返回空结果 for (const idxLsm of this.secondaryIndexes.values()) { await idxLsm.flush(); } // v0.4.5: 页面化存储 — 落盘全部脏页(save 已逐页落盘,此处兜底) await this.bufferPool.flushAll(); await this.wal.flush(); // v0.4.2-fix: close 前 checkpoint(截断 WAL)— // 此前只 flush 不截断,下次打开会重放全部历史 WAL 记录(含已落盘 SSTable 的数据), // 重复解析/重复 put 拖慢启动,并与恢复后 flush+checkpoint 竞争放大丢数据 await this.wal.checkpoint(); await this.backend.close(); // v0.4.5: 释放多标签页独占锁(等待锁真正归还) if (this.dbLock) { await this.dbLock.release(); this.dbLock = null; } // v0.4.2-fix: 清空运行期状态(此前 close 后 mvcc/txn 残留, // 重开时 beginTransaction 报 TX_ACTIVE 或读到陈旧快照) this.schemas.clear(); this.tablePKs.clear(); this.secondaryIndexes.clear(); this.uniqueIndexCols.clear(); this.mvcc = new MVCCManager(); this.currentTxnId = null; this.txnSnapshot = null; this.savepoints.clear(); this.opCounter = 0; this.opened = false; } /** * v0.4.2-fix: 崩溃恢复/自愈 — 校验并移除损坏 SSTable、截断 WAL、重建二级索引。 * v0.4.5 增强:清理 OPFS 残留临时文件、清理孤儿页面(meta 未引用的 pg_ 文件)。 * 应用层检测到异常后调用,无需删库重建。 */ async repair(): Promise { this.ensureOpen(); // v0.6.0-fix: 先清页面缓存再校验 — 缓存中的"完好页面"会掩盖磁盘损坏 await this.bufferPool.clear(); // 1. 校验全部 SSTable,移除残缺项(打开时已做一次,此处兜底运行期损坏) const removed = await this.lsm.validateAll(); // 2. 将 WAL 残留数据落盘并截断,避免无限重放(含空洞截断落地) await this.lsm.flush(); await this.wal.checkpoint(); // 3. 重建所有表的二级索引(修复索引与主数据不一致) for (const tableName of this.schemas.keys()) { await this.reindexTable(tableName); } // v0.4.5: 4. 清理 OPFS 残留临时文件(.crswap/.tmp) const backendAny = this.backend as unknown as { cleanupStaleFiles?: () => Promise }; if (typeof backendAny.cleanupStaleFiles === 'function') { try { await backendAny.cleanupStaleFiles(); } catch { /* 清理失败不阻塞 */ } } // v0.4.5: 5. 清理孤儿页面(所有 LSM 命名空间 meta 均未引用的 pg_ 文件) await this.cleanupOrphanPages(); if (removed > 0) { // eslint-disable-next-line no-console console.warn(`[AriaEngine] repair: removed ${removed} corrupted SSTable(s)`); } } /** * v0.4.5: 清理孤儿页面 — 扫描全部 pg_* 文件,未被任何 LSM 命名空间 meta 引用的删除。 * 孤儿页面来自:崩溃中断的 compaction/删除流程(旧 SSTable 页面残留)。 */ private async cleanupOrphanPages(): Promise { const keys = await this.backend.listKeys(); const pgKeys = keys.filter((k) => /^pg_\d+$/.test(k)); if (pgKeys.length === 0) return; const used = new Set(); const collectMeta = async (ns: string): Promise => { const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`; const raw = await this.backend.read(META_KEY); if (!raw) return; try { const metas = JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[]; for (const m of metas) { if (m.pageIds) { for (const pid of m.pageIds) used.add(pid); } } } catch { /* 损坏的 meta 忽略(validateAll 已处理) */ } }; await collectMeta('main'); // 收集全部二级索引命名空间 for (const [tableName, schema] of this.schemas) { for (const [colName, colDef] of Object.entries(schema.columns)) { if (colDef.index || colDef.unique) { await collectMeta(`idx_${tableName}_${colName}`); } } } const orphanIds = pgKeys .map((k) => Number(k.slice('pg_'.length))) .filter((pid) => !used.has(pid)); if (orphanIds.length > 0) { await this.backend.deleteMany(orphanIds.map((pid) => `pg_${pid}`)); } } /** * v0.4.1: 重置数据库 — 清空全部数据与表结构(演示页刷新/重新初始化用)。 * 清空存储后端、LSM、WAL、MVCC 与二级索引,后续可继续使用本实例。 */ async clearAll(): Promise { this.ensureOpen(); // 清空存储后端(页面文件 / WAL 记录 / schema 记录 / 元数据) await this.backend.clear(); // v0.4.5: 清空页面缓存与页面 ID 分配状态 await this.bufferPool.clear(); await this.fileManager.clearAll(); this.schemas.clear(); this.tablePKs.clear(); this.secondaryIndexes.clear(); this.uniqueIndexCols.clear(); this.lsm.clear(); this.mvcc = new MVCCManager(); this.currentTxnId = null; this.txnSnapshot = null; this.savepoints.clear(); this.opCounter = 0; // 持久化空 schema(防止旧 schema 记录残留) await this.persistSchemas(); // 重置 WAL 状态(backend.clear 已清记录,同步内存计数) await this.wal.checkpoint(); } isOpen(): boolean { return this.opened; } // ---- v0.4.2-fix: 库内元数据(迁移版本持久化用) ---- async getMeta(key: string): Promise { const raw = await this.backend.read(`__meta_${key}`); return raw ? new TextDecoder().decode(raw) : null; } async setMeta(key: string, value: string): Promise { await this.backend.write(`__meta_${key}`, new TextEncoder().encode(value).buffer); } // ======================================================================= // 表管理 // ======================================================================= async createTable(schema: TableSchema): Promise { this.ensureOpen(); // v0.4.2-fix: Aria 事务中 DDL 显式拒绝(事务快照只覆盖行数据, // 结构变更无法回滚;Memory/IndexedDB 引擎快照可回滚,行为不一致 → 明确报错而非静默) this.ensureNoDDLInTransaction('CREATE TABLE'); if (this.schemas.has(schema.name)) { throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS'); } this.schemas.set(schema.name, schema); this.tablePKs.set(schema.name, this.getPK(schema)); // 为索引列创建二级索引 LSM(每个索引使用独立命名空间的 SSTableStore,避免 id/meta 冲突) // v0.3.3: 主键列不建冗余二级索引(主 LSM 本身就是 PK 索引,范围查询走前缀扫描) for (const [colName, colDef] of Object.entries(schema.columns)) { if (colDef.index || colDef.unique) { const idxKey = `${schema.name}:idx:${colName}`; if (!this.secondaryIndexes.has(idxKey)) { const idxLsm = new LSM({ memtableSizeThreshold: this.config.memtableSizeThreshold, levelSizeMultiplier: this.config.levelSizeMultiplier, blockSize: this.config.pageSize, bloomBitsPerKey: this.config.bloomFilterBitsPerKey, cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, sstableStore: this.createSSTableStore(`idx_${schema.name}_${colName}`), }); await idxLsm.init(); this.secondaryIndexes.set(idxKey, idxLsm); } } } await this.persistSchemas(); await this.wal.append({ type: WALRecordType.CREATE_TABLE, txnId: 0, tableName: schema.name, key: '', data: { schema: JSON.stringify(schema) } as unknown as Record, }); } async dropTable(tableName: string): Promise { this.ensureOpen(); this.ensureNoDDLInTransaction('DROP TABLE'); this.ensureTable(tableName); // 删除表中所有行 const rows = await this.getAllRows(tableName); for (const row of rows) { const pkCol = this.tablePKs.get(tableName)!; this.lsm.delete(`${tableName}:${row[pkCol]}`); } // v0.4.2-fix: 清理该表的全部二级索引 LSM 与持久化文件 — // 此前残留孤儿索引,重建同名表后旧索引数据污染新表(索引查询返回错误行) await this.cleanupTableIndexes(tableName); // v0.7.4: 清理该表的 unique 索引来源标记 const uPrefix = `${tableName}:`; for (const k of this.uniqueIndexCols) { if (k.startsWith(uPrefix)) this.uniqueIndexCols.delete(k); } this.schemas.delete(tableName); this.tablePKs.delete(tableName); await this.persistSchemas(); await this.wal.append({ type: WALRecordType.DROP_TABLE, txnId: 0, tableName, key: '', }); } /** * v0.4.2-fix: 清理指定表的全部二级索引 LSM(内存 + 存储文件 + meta)。 * dropTable / DROP_TABLE 恢复 / alterTable DROP 索引列 共用。 */ private async cleanupTableIndexes(tableName: string): Promise { const prefix = `${tableName}:idx:`; const toDelete: string[] = []; for (const [idxKey, idxLsm] of this.secondaryIndexes) { if (!idxKey.startsWith(prefix)) continue; toDelete.push(idxKey); try { await idxLsm.clear(); } catch { /* 清理失败不阻塞 */ } } for (const idxKey of toDelete) { this.secondaryIndexes.delete(idxKey); } } async hasTable(tableName: string): Promise { // v0.7.1: 未 open 防护统一(此前与 KVStoreEngine 不一致:返回空而非报错) this.ensureOpen(); return this.schemas.has(tableName); } async getTableNames(): Promise { this.ensureOpen(); return Array.from(this.schemas.keys()); } async getTableSchema(tableName: string): Promise { this.ensureOpen(); return this.schemas.get(tableName) ?? null; } // ======================================================================= // CRUD // ======================================================================= async insert(tableName: string, rows: Record[]): Promise { this.ensureOpen(); this.ensureTable(tableName); const schema = this.schemas.get(tableName)!; const pkCol = this.tablePKs.get(tableName)!; const pks: string[] = []; // v0.3.1: 批量 WAL 写入(组提交),一次 insert 合并为一次落盘 const walRecords: Omit[] = []; // v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable(一次 drainChain)。 // 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链, // 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖 // (10 万行 kv 后端从 5ms/批暴跌到 8~11s/批)。批内新数据在 memtable // 或 flush 产物(自动入缓存),循环内 lsm.get 始终完整。 // // v0.6.2: 整批预校验(验证失败整批不落库,语义更原子)+ 唯一约束检查。 const uniqueCols = this.uniqueColumns(tableName, schema); const validatedRows: { row: Record; pkValue: string; key: string }[] = []; for (const row of rows) { const validated = this.validateRow(schema, row); const pkValue = String(validated[pkCol]); validatedRows.push({ row: validated, pkValue, key: `${tableName}:${pkValue}` }); } await this.lsm.prefetchKeys(validatedRows.map((v) => v.key)); // v0.7.3: 主键批内互查 + 预检 —— 此前 PK 重复检查在写入循环内: // 第 N 行重复抛错时,前 N-1 行已 put LSM 且其 WAL 记录随 appendBatch 一起 // 丢失 → 语句级部分提交 + 内存/WAL 不一致(与 v0.6.2 的 unique 预检同一阶段)。 const pkSet = new Set(); for (const { pkValue, key } of validatedRows) { if (pkSet.has(pkValue)) { throw new DatabaseError( `Duplicate primary key "${pkValue}" in table "${tableName}"`, 'DUPLICATE_KEY', ); } pkSet.add(pkValue); // v0.8.0: 事务快照优先,否则回源 LSM(lsm.get 现为 async) const existing = this.currentTxnId ? (this.txnSnapshot?.get(key) ?? await this.lsm.get(key)) : await this.lsm.get(key); if (existing && !(existing as unknown as Record).__txn_deleted) { throw new DatabaseError( `Duplicate primary key "${pkValue}" in table "${tableName}"`, 'DUPLICATE_KEY', ); } } // v0.6.2: 唯一约束 — 批量预加载本批唯一列涉及的索引范围(一次 drainChain) for (const colName of uniqueCols) { const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`)!; await idxLsm.prefetchPrefixRanges( validatedRows .map((v): [string, string] | null => { const val = v.row[colName]; if (val === undefined || val === null) return null; const p = `${String(val)}:`; return [p, `${p}\uffff`]; }) .filter((r): r is [string, string] => r !== null), ); } // v0.6.2: 唯一性整批预检(批内互查 + 索引查)—— 失败整批不落库(原子语义) const batchUnique = new Map>(); for (const { row: validated, pkValue } of validatedRows) { for (const colName of uniqueCols) { const val = validated[colName]; if (val === undefined || val === null) continue; const v = String(val); let seen = batchUnique.get(colName); if (!seen) { seen = new Set(); batchUnique.set(colName, seen); } if (seen.has(v)) { throw new DatabaseError( `Unique constraint violation on column "${colName}" in table "${tableName}"`, 'UNIQUE_VIOLATION', ); } seen.add(v); await this.checkUnique(tableName, [colName], validated, pkValue); } } for (const { row: validated, pkValue, key } of validatedRows) { // PK 重复已在批预检阶段检查(v0.7.3),此处不再重复查询 if (this.currentTxnId && this.txnSnapshot) { // Within transaction: buffer to snapshot + MVCC version chain this.txnSnapshot.set(key, validated); this.mvcc.writeVersion(tableName, pkValue, validated, this.currentTxnId); } else { // Direct write to LSM (PK index) this.lsm.put(key, validated); } // 更新二级索引 this.updateSecondaryIndexes(tableName, pkValue, validated, null); pks.push(pkValue); walRecords.push({ type: WALRecordType.INSERT, txnId: this.currentTxnId ?? 0, tableName, key: pkValue, data: validated, }); } await this.wal.appendBatch(walRecords); this.txnWalRecordCount += walRecords.length; this.opCounter += rows.length; this.checkMemoryBudget(); await this.checkpointManager.tick(); this.tryGC(); return pks; } async find(tableName: string, query: QueryPlan): Promise[]> { this.ensureOpen(); this.ensureTable(tableName); let rows: Record[]; // Try index lookup const fastPath = await this.tryIndexLookup(tableName, query); if (fastPath !== null) { rows = fastPath; } else { rows = await this.getAllRows(tableName); } // v0.3.3: 事务内合并未提交快照(统一在 mergeTxnSnapshot 处理) rows = this.mergeTxnSnapshot(tableName, rows); // WHERE filter if (query.where && Object.keys(query.where).length > 0) { rows = rows.filter((row) => matchWhere(row, query.where!)); } // ORDER if (query.orderBy && query.orderBy.length > 0) { rows = applyOrderBy(rows, query.orderBy); } // LIMIT/OFFSET const offset = query.offset ?? 0; const limit = query.limit ?? rows.length; rows = rows.slice(offset, offset + limit); // Column projection if (query.columns && query.columns.length > 0 && query.columns[0] !== '*') { rows = rows.map((row) => projectColumns(row, query.columns!)); } // 查询完成,回收查询期间的临时缓存超限 this.trimAllCaches(); // v0.8.0: 行所有权 —— 返回副本,调用方不得改写存储(见 engine/interface.ts 约定) return rows.map((row) => cloneRow(row)); } async update( tableName: string, query: QueryPlan, updates: Record, ): Promise { this.ensureOpen(); this.ensureTable(tableName); // v0.7.4: 防御 —— QueryBuilder 直通引擎不经 Executor 子查询解析, // 未解析的 $subquery/$col/$exists 在 matchWhere 中恒 false → 静默 0 行 if (containsUnresolvedSubqueries(query.where)) { throw new DatabaseError( 'Unresolved subqueries/column references in UPDATE WHERE (use db.query() to execute subqueries)', 'NOT_SUPPORTED', ); } const schema = this.schemas.get(tableName)!; const rows = await this.getAllRows(tableName); let count = 0; // v0.3.1: 批量 WAL 写入(组提交) const walRecords: Omit[] = []; // v0.4.2-fix: ON UPDATE 级联环路保护 const visited = new Set(); // v0.7.2: undefined 值视为"不更新该列"(保留旧值),null 显式置空 const cleanUpdates = stripUndefinedUpdates(updates); // v0.7.4: 未知列显式报错 —— 此前 SET nonexistent = ... 被静默写入存储行 // (validateRow 只遍历 schema 列,脏列残留在行内并随 SSTable 持久化) for (const col of Object.keys(cleanUpdates)) { if (!schema.columns[col]) { throw new DatabaseError(`Column "${col}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND'); } } // v0.6.2: 唯一约束 — 批量预加载本批更新涉及的唯一列索引范围(一次 drainChain) const uniqueCols = this.uniqueColumns(tableName, schema); for (const colName of uniqueCols) { const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`)!; const ranges: [string, string][] = []; if (cleanUpdates[colName] !== undefined && cleanUpdates[colName] !== null) { const p = `${String(cleanUpdates[colName])}:`; ranges.push([p, `${p}\uffff`]); } else if (!(colName in cleanUpdates)) { for (const row of rows) { const val = row[colName]; if (val === undefined || val === null) continue; const p = `${String(val)}:`; ranges.push([p, `${p}\uffff`]); } } await idxLsm.prefetchPrefixRanges(ranges); } // v0.7.2: 语句级原子性 — 两阶段(先全量预检,后执行)。 // 此前逐行"校验+写入":第 N 行唯一冲突/校验失败抛错时,前 N-1 行已写入 // 且其 WAL 记录随 appendBatch 一起丢失 → 内存已改、WAL 无记录、调用方已收到错误 // (无事务下语句级部分提交 + 崩溃后进一步不一致)。 const planned: { row: Record; pk: string; key: string; updated: Record; newPk: string; pkChanged: boolean }[] = []; const batchUnique: Map> = new Map(); // 阶段 1:全量预检(任何一行失败 → 整条语句不执行) for (const row of rows) { const pkCol = this.tablePKs.get(tableName)!; const key = `${tableName}:${row[pkCol]}`; if (query.where && Object.keys(query.where).length > 0 && !matchWhere(row, query.where)) continue; const updated = { ...row, ...cleanUpdates }; this.validateRow(schema, updated); // 批内唯一互查(索引尚未更新,两行同时改到同一新值需要互查兜底) this.checkBatchUnique(tableName, uniqueCols, updated, batchUnique); // v0.6.2: 唯一约束检查(排除自身旧索引条目:主键变更时旧条目仍以旧键存在) await this.checkUnique(tableName, uniqueCols, updated, String(row[pkCol])); // v0.4.2-fix: 支持更新主键 — 删除旧键 + 落新键 + WAL 两条记录 const newPk = String(updated[pkCol]); const pkChanged = newPk !== String(row[pkCol]); // v0.6.2-fix(P0): 主键变更撞已有主键 → 抛 DUPLICATE_KEY // (此前静默覆盖另一行丢数据;与 MemoryEngine 对齐) if (pkChanged) { const newKey = `${tableName}:${newPk}`; const existing = this.currentTxnId ? (this.txnSnapshot?.get(newKey) ?? await this.lsm.get(newKey)) : await this.lsm.get(newKey); if (existing && !(existing as unknown as Record).__txn_deleted) { throw new DatabaseError( `Duplicate primary key "${newPk}" in table "${tableName}" (cannot update key to existing value)`, 'DUPLICATE_KEY', ); } } planned.push({ row, pk: String(row[pkCol]), key, updated, newPk, pkChanged }); } // 阶段 1b:主键变更 RESTRICT / SET NULL+required 预检(任何修改前) for (const p of planned) { if (p.pkChanged) { await this.checkForeignKeyUpdateRestrict(tableName, p.pk, p.newPk); } } // 阶段 2:执行(预检已通过,此阶段不再抛校验类错误) for (const { row, pk, key, updated, newPk, pkChanged } of planned) { if (pkChanged) { // ON UPDATE 外键级联(RESTRICT 抛错 / CASCADE / SET NULL) await this.applyForeignKeyUpdateRules(tableName, pk, newPk, walRecords, visited); } if (this.currentTxnId && this.txnSnapshot) { if (pkChanged) { this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record); this.mvcc.deleteVersion(tableName, pk, this.currentTxnId); } this.txnSnapshot.set(`${tableName}:${newPk}`, updated); this.mvcc.writeVersion(tableName, newPk, updated, this.currentTxnId); } else { if (pkChanged) this.lsm.delete(key); this.lsm.put(`${tableName}:${newPk}`, updated); } count++; if (pkChanged) { walRecords.push({ type: WALRecordType.DELETE, txnId: this.currentTxnId ?? 0, tableName, key: pk, }); } walRecords.push({ type: WALRecordType.UPDATE, txnId: this.currentTxnId ?? 0, tableName, key: newPk, data: updated, }); // 更新二级索引(主键变更时旧索引条目一并清理) // v0.6.2-fix: 此前非主键更新不传旧行 → 旧索引条目残留 // (唯一性检查误报 / 索引存储膨胀);现在统一传旧行清理旧值 this.updateSecondaryIndexes(tableName, newPk, updated, row); } await this.wal.appendBatch(walRecords); this.txnWalRecordCount += walRecords.length; this.opCounter += count; await this.checkpointManager.tick(); this.trimAllCaches(); return count; } /** * v0.7.2: 批内唯一互查 — 两条行在同一语句中更新到同一唯一值时的兜底检查 * (阶段 1 中索引尚未反映本语句的变更)。 */ private checkBatchUnique( tableName: string, uniqueCols: string[], updated: Record, batchUnique: Map>, ): void { for (const colName of uniqueCols) { const value = updated[colName]; if (value === undefined || value === null) continue; let seen = batchUnique.get(colName); if (!seen) { seen = new Set(); batchUnique.set(colName, seen); } if (seen.has(value)) { throw new DatabaseError( `Unique constraint violation on column "${colName}" in table "${tableName}"`, 'UNIQUE_VIOLATION', ); } seen.add(value); } } /** * v0.7.2: ON UPDATE 外键预检 — 从 applyForeignKeyUpdateRules 提取(两阶段 update 用): * RESTRICT 存在依赖行抛错;SET NULL 撞 required 列同样整体拒绝。 */ private async checkForeignKeyUpdateRestrict(tableName: string, oldPk: string, _newPk: string): Promise { for (const [refTableName, refSchema] of this.schemas) { if (refTableName === tableName) continue; for (const [colName, colDef] of Object.entries(refSchema.columns)) { if (!colDef.references || !colDef.onUpdate) continue; const [refTable] = colDef.references.split('.'); if (refTable !== tableName) continue; if (colDef.onUpdate === 'RESTRICT' || (colDef.onUpdate === 'SET NULL' && colDef.required)) { const refRows = await this.getAllRows(refTableName); for (const refRow of refRows) { if (String(refRow[colName]) === oldPk) { const reason = colDef.onUpdate === 'RESTRICT' ? `foreign key "${colName}" in "${refTableName}" has dependent rows` : `foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`; throw new DatabaseError( `Cannot update "${tableName}" key "${oldPk}": ${reason}`, 'FOREIGN_KEY_VIOLATION', ); } } } } } } /** * v0.4.2-fix: ON UPDATE 外键级联 — 主键 oldPk → newPk 时处理引用表。 * RESTRICT 抛错 / CASCADE 更新 FK / SET NULL 置空(含索引与 WAL 记录)。 * 两阶段:先全量 RESTRICT 检查,再执行级联。 */ private async applyForeignKeyUpdateRules( tableName: string, oldPk: string, newPk: string, walRecords: Omit[], visited: Set, ): Promise { const visitKey = `${tableName}:${oldPk}`; if (visited.has(visitKey)) return; visited.add(visitKey); // v0.7.3-perf: 删除冗余的阶段 1 RESTRICT 扫描 —— checkForeignKeyUpdateRestrict // 已在两阶段 update 预检(阶段 1b)覆盖 RESTRICT 与 SET NULL+required, // 此处任何修改前重复全表扫描纯属浪费。直接执行 CASCADE / SET NULL。 for (const [refTableName, refSchema] of this.schemas) { if (refTableName === tableName) continue; for (const [colName, colDef] of Object.entries(refSchema.columns)) { if (!colDef.references || !colDef.onUpdate) continue; const [refTable] = colDef.references.split('.'); if (refTable !== tableName) continue; if (colDef.onUpdate !== 'CASCADE' && colDef.onUpdate !== 'SET NULL') continue; const refRows = await this.getAllRows(refTableName); for (const refRow of refRows) { if (String(refRow[colName]) !== oldPk) continue; const refPkCol = this.tablePKs.get(refTableName)!; const refPk = String(refRow[refPkCol]); const updatedRef = { ...refRow, [colName]: colDef.onUpdate === 'CASCADE' ? newPk : null }; const refKey = `${refTableName}:${refPk}`; if (this.currentTxnId && this.txnSnapshot) { this.txnSnapshot.set(refKey, updatedRef); this.mvcc.writeVersion(refTableName, refPk, updatedRef, this.currentTxnId); } else { this.lsm.put(refKey, updatedRef); } this.updateSecondaryIndexes(refTableName, refPk, updatedRef, refRow); walRecords.push({ type: WALRecordType.UPDATE, txnId: this.currentTxnId ?? 0, tableName: refTableName, key: refPk, data: updatedRef, }); } } } } async delete(tableName: string, query: QueryPlan): Promise { this.ensureOpen(); this.ensureTable(tableName); // v0.7.4: 防御 —— QueryBuilder 直通引擎不经 Executor 子查询解析, // 未解析的 $subquery/$col/$exists 在 matchWhere 中恒 false → 静默 0 行 if (containsUnresolvedSubqueries(query.where)) { throw new DatabaseError( 'Unresolved subqueries/column references in DELETE WHERE (use db.query() to execute subqueries)', 'NOT_SUPPORTED', ); } const rows = await this.getAllRows(tableName); let count = 0; // v0.3.1: 批量 WAL 写入(组提交) const walRecords: Omit[] = []; // v0.4.1: 外键级联(环路保护) const visited = new Set(); // v0.6.3-fix: 级联两阶段 —— 先对全部匹配行做 RESTRICT 预检(沿 CASCADE 链递归), // 否则第 N 行 RESTRICT 抛错时前 N-1 行的级联已执行 → 无事务部分级联(数据不一致) const matchedPks: string[] = []; for (const row of rows) { if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) { matchedPks.push(String(row[this.tablePKs.get(tableName)!])); } } const restrictVisited = new Set(); for (const pkValue of matchedPks) { await this.checkCascadeRestrict(tableName, pkValue, restrictVisited); } for (const row of rows) { const pkCol = this.tablePKs.get(tableName)!; const key = `${tableName}:${row[pkCol]}`; if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) { // v0.4.1: 外键规则(RESTRICT 抛错 / CASCADE 递归删 / SET NULL 置空) count += await this.applyForeignKeyRules(tableName, String(row[pkCol]), walRecords, visited); if (this.currentTxnId && this.txnSnapshot) { // Buffer delete in snapshot + MVCC tombstone this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record); this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId); } else { this.lsm.delete(key); } count++; walRecords.push({ type: WALRecordType.DELETE, txnId: this.currentTxnId ?? 0, tableName, key: String(row[pkCol]), }); // 移除二级索引 this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row); } } await this.wal.appendBatch(walRecords); this.txnWalRecordCount += walRecords.length; this.opCounter += count; await this.checkpointManager.tick(); this.trimAllCaches(); return count; } /** * v0.6.3: RESTRICT 预检(delete 级联两阶段之一,与 MemoryEngine 对齐)。 * 递归沿 CASCADE 链检查引用表:RESTRICT 引用存在依赖行则抛 FOREIGN_KEY_VIOLATION。 */ private async checkCascadeRestrict(tableName: string, pkValue: string, visited: Set): Promise { const visitKey = `${tableName}:${pkValue}`; if (visited.has(visitKey)) return; visited.add(visitKey); for (const [refTableName, refSchema] of this.schemas) { if (refTableName === tableName) continue; for (const [colName, colDef] of Object.entries(refSchema.columns)) { if (!colDef.references || !colDef.onDelete) continue; const [refTable] = colDef.references.split('.'); if (refTable !== tableName) continue; const refRows = await this.getAllRows(refTableName); const matched = refRows.filter((r) => String(r[colName]) === pkValue); if (colDef.onDelete === 'RESTRICT' && matched.length > 0) { throw new DatabaseError( `Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION', ); } // v0.7.2: SET NULL 到 required 列违反约束 —— 预检阶段整体拒绝 if (colDef.onDelete === 'SET NULL' && colDef.required && matched.length > 0) { throw new DatabaseError( `Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`, 'FOREIGN_KEY_VIOLATION', ); } if (colDef.onDelete === 'CASCADE') { const refPkCol = this.tablePKs.get(refTableName)!; for (const refRow of matched) { await this.checkCascadeRestrict(refTableName, String(refRow[refPkCol]), visited); } } } } } /** * v0.4.1: 外键级联规则 — 对齐 MemoryEngine.cascadeDelete 行为。 * 删除 tableName 主键为 pkValue 的行前,检查引用它的所有表: * - RESTRICT: 存在引用行 → 抛 FOREIGN_KEY_VIOLATION * - CASCADE: 递归删除引用行(含索引/WAL) * - SET NULL: 引用行外键列置 null(含索引/WAL) * @returns 级联影响的行数(CASCADE 删除行数 + SET NULL 更新行数) */ private async applyForeignKeyRules( tableName: string, pkValue: string, walRecords: Omit[], visited: Set, ): Promise { let total = 0; const visitKey = `${tableName}:${pkValue}`; if (visited.has(visitKey)) return 0; visited.add(visitKey); for (const [refTableName, refSchema] of this.schemas) { if (refTableName === tableName) continue; for (const [colName, colDef] of Object.entries(refSchema.columns)) { if (!colDef.references || !colDef.onDelete) continue; const [refTable] = colDef.references.split('.'); if (refTable !== tableName) continue; const refRows = await this.getAllRows(refTableName); const matched = refRows.filter((r) => String(r[colName]) === pkValue); if (colDef.onDelete === 'RESTRICT' && matched.length > 0) { throw new DatabaseError( `Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION', ); } if (colDef.onDelete === 'CASCADE') { const refPkCol = this.tablePKs.get(refTableName)!; for (const refRow of matched) { const refPk = String(refRow[refPkCol]); // 递归级联(先处理更深层引用) total += await this.applyForeignKeyRules(refTableName, refPk, walRecords, visited); // 删除引用行 const refKey = `${refTableName}:${refPk}`; if (this.currentTxnId && this.txnSnapshot) { this.txnSnapshot.set(refKey, { __txn_deleted: true } as unknown as Record); this.mvcc.deleteVersion(refTableName, refPk, this.currentTxnId); } else { this.lsm.delete(refKey); } this.updateSecondaryIndexes(refTableName, refPk, null, refRow); walRecords.push({ type: WALRecordType.DELETE, txnId: this.currentTxnId ?? 0, tableName: refTableName, key: refPk, }); total++; } } else if (colDef.onDelete === 'SET NULL') { const refPkCol = this.tablePKs.get(refTableName)!; for (const refRow of matched) { const refPk = String(refRow[refPkCol]); const updated = { ...refRow, [colName]: null }; const refKey = `${refTableName}:${refPk}`; if (this.currentTxnId && this.txnSnapshot) { this.txnSnapshot.set(refKey, updated); this.mvcc.writeVersion(refTableName, refPk, updated, this.currentTxnId); } else { this.lsm.put(refKey, updated); } this.updateSecondaryIndexes(refTableName, refPk, updated, refRow); walRecords.push({ type: WALRecordType.UPDATE, txnId: this.currentTxnId ?? 0, tableName: refTableName, key: refPk, data: updated, }); // 对齐 Memory 语义:SET NULL 不影响返回的删除行数 } } } } return total; } /** * v0.4.0: 流式查询 — 逐行回调,不物化结果数组。 * 全表路径走 LSM rangeScanLazy 惰性扫描;索引等值/范围路径复用 tryIndexLookup。 * 事务中回退物化(快照合并需要全量行集)。 */ async findStream( tableName: string, query: QueryPlan, onRow: (row: Record) => void, ): Promise { this.ensureOpen(); this.ensureTable(tableName); const hasWhere = !!(query.where && Object.keys(query.where).length > 0); const project = query.columns && query.columns.length > 0 && query.columns[0] !== '*' ? (row: Record) => projectColumns(row, query.columns!) : null; const limit = query.limit ?? Infinity; const offset = query.offset ?? 0; const pkCol = this.tablePKs.get(tableName)!; const prefix = `${tableName}:`; let count = 0; let skipped = 0; const emit = (row: Record): boolean => { if (hasWhere && !matchWhere(row, query.where!)) return true; if (skipped < offset) { skipped++; return true; } onRow(project ? project(row) : cloneRow(row)); count++; return count < limit; }; if (this.currentTxnId && this.txnSnapshot) { // 事务中:物化后逐行回调(快照合并需要全量行集) const rows = await this.find(tableName, { ...query, orderBy: undefined, limit: undefined, offset: undefined }); for (const row of rows) { onRow(project ? project(row) : cloneRow(row)); } return rows.length; } // 索引路径:等值/范围查找(结果行已过滤,直接回调) const fastPath = await this.tryIndexLookup(tableName, query); if (fastPath !== null) { for (const row of fastPath) { if (!emit(row)) break; } return count; } // 全表惰性扫描(含 WHERE 过滤,不物化;v0.7.4: callback 返回 false 提前终止, // 未消费的 SSTable 块 / 子树不再解析 —— 真流式,大表 limit 内存 O(1)) await this.lsm.prefetchRange(prefix, `${prefix}\uffff`); await this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => { if (count >= limit) return false; const row = { ...value }; row[pkCol] = key.slice(prefix.length); return emit(row); }); return count; } async count(tableName: string, query?: QueryPlan): Promise { this.ensureOpen(); this.ensureTable(tableName); const rows = await this.getAllRows(tableName); this.trimAllCaches(); if (!query?.where || Object.keys(query.where).length === 0) return rows.length; return rows.filter((row) => matchWhere(row, query.where!)).length; } async clear(tableName: string): Promise { this.ensureOpen(); this.ensureTable(tableName); const rows = await this.getAllRows(tableName); // v0.3.3: 事务内清空走快照(删除标记),提交时生效;并写入 WAL const walRecords: Omit[] = []; for (const row of rows) { const pkCol = this.tablePKs.get(tableName)!; const key = `${tableName}:${row[pkCol]}`; if (this.currentTxnId && this.txnSnapshot) { this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record); this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId); } else { this.lsm.delete(key); } walRecords.push({ type: WALRecordType.DELETE, txnId: this.currentTxnId ?? 0, tableName, key: String(row[pkCol]), }); // 移除二级索引 this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row); } await this.wal.appendBatch(walRecords); this.txnWalRecordCount += walRecords.length; this.opCounter += rows.length; await this.checkpointManager.tick(); this.tryGC(); } // ---- ALTER TABLE(v0.4.1) ---- /** * v0.4.1: ALTER TABLE — 结构变更真正生效于存储: * - ADD: 持久化 schema(persistSchemas),行无需修改 * - DROP: 持久化 schema + 遍历主 LSM 重写所有行(移除该列键)+ WAL UPDATE 记录 * (通用路径 getTableSchema 返回副本,Executor 的引用修改对 Aria 无效) */ async alterTable( tableName: string, action: 'ADD' | 'DROP', column: import('../../constants').ColumnDef & { name: string }, ): Promise { this.ensureOpen(); this.ensureNoDDLInTransaction('ALTER TABLE'); this.ensureTable(tableName); const schema = this.schemas.get(tableName)!; if (action === 'ADD') { if (schema.columns[column.name]) { throw new DatabaseError(`Column "${column.name}" already exists in table "${tableName}"`, 'COLUMN_EXISTS'); } schema.columns[column.name] = column; await this.persistSchemas(); return; } // DROP if (!schema.columns[column.name]) { throw new DatabaseError(`Column "${column.name}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND'); } // v0.4.2-fix: 被删列是索引列 → 先清理索引 LSM(残留会导致后续同名列索引脏数据) if (schema.columns[column.name].index || schema.columns[column.name].unique) { const idxKey = `${tableName}:idx:${column.name}`; const idxLsm = this.secondaryIndexes.get(idxKey); if (idxLsm) { try { await idxLsm.clear(); } catch { /* 清理失败不阻塞 */ } this.secondaryIndexes.delete(idxKey); } } delete schema.columns[column.name]; await this.persistSchemas(); // 重写主 LSM:移除所有行的该列键(find 副本无法就地删除,必须重写存储) const prefix = `${tableName}:`; const endKey = `${prefix}\uffff`; await this.lsm.prefetchRange(prefix, endKey); const entries = await this.lsm.rangeScan(prefix, endKey); const walRecords: Omit[] = []; for (const [key, value] of entries) { if (!(column.name in value)) continue; const updated = { ...value }; delete updated[column.name]; this.lsm.put(key, updated); // 二级索引列被删时同步清理索引 const pk = key.slice(prefix.length); this.updateSecondaryIndexes(tableName, pk, updated, value); walRecords.push({ type: WALRecordType.UPDATE, txnId: this.currentTxnId ?? 0, tableName, key: pk, data: updated, }); } await this.wal.appendBatch(walRecords); this.txnWalRecordCount += walRecords.length; this.opCounter += walRecords.length; await this.checkpointManager.tick(); this.trimAllCaches(); } async createIndex(tableName: string, column: string, unique?: boolean): Promise { this.ensureOpen(); this.ensureNoDDLInTransaction('CREATE INDEX'); this.ensureTable(tableName); const schema = this.schemas.get(tableName)!; const colDef = schema.columns[column]; if (!colDef) throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND'); const idxKey = `${tableName}:idx:${column}`; // v0.4.2-fix: 以索引 LSM 是否已建为准(schema 标记可能因重启恢复而存在, // 但索引 LSM 未恢复 → 此前静默 return 导致索引永久缺失) if (this.secondaryIndexes.has(idxKey)) return; const idxLsm = new LSM({ memtableSizeThreshold: this.config.memtableSizeThreshold, levelSizeMultiplier: this.config.levelSizeMultiplier, blockSize: this.config.pageSize, bloomBitsPerKey: this.config.bloomFilterBitsPerKey, cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize, sstableStore: this.createSSTableStore(`idx_${tableName}_${column}`), }); await idxLsm.init(); this.secondaryIndexes.set(idxKey, idxLsm); try { // 从主 LSM 重建索引数据 const pkCol = this.tablePKs.get(tableName)!; const rows = await this.getAllRows(tableName); const seen = new Set(); for (const row of rows) { const value = row[column]; if (value !== undefined && value !== null) { const v = String(value); // v0.7.3: UNIQUE 索引回填校验存量唯一性 —— 此前重复数据静默建索引 // (SQLite 语义应报错),与 MemoryEngine 对齐 if (unique && seen.has(v)) { throw new DatabaseError( `Unique index on column "${column}" in table "${tableName}" cannot be created: duplicate value "${v}"`, 'UNIQUE_VIOLATION', ); } seen.add(v); idxLsm.put(`${v}:${row[pkCol]}`, { pk: row[pkCol] }); } } await idxLsm.flush(); } catch (error) { // 回填失败(唯一冲突):清理半初始化索引(内存 + 存储),标志未落,保持原子语义 this.secondaryIndexes.delete(idxKey); try { await idxLsm.clear(); } catch { /* 清理失败不阻塞 */ } throw error; } colDef.index = true; if (unique) { colDef.unique = true; // v0.7.4: 记录唯一约束来源(DROP INDEX 时可解除;建表约束不可) this.uniqueIndexCols.add(`${tableName}:${column}`); } await this.persistSchemas(); } async dropIndex(tableName: string, column: string, _indexName?: string): Promise { this.ensureOpen(); this.ensureNoDDLInTransaction('DROP INDEX'); this.ensureTable(tableName); const schema = this.schemas.get(tableName)!; const colDef = schema.columns[column]; if (!colDef) throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND'); // 主键索引不可删除(PK 查找依赖主 LSM) if (colDef.primaryKey) { throw new DatabaseError(`Cannot drop primary key index on column "${column}"`, 'NOT_SUPPORTED'); } // v0.4.1: DROP 不存在的索引应报错(此前静默成功) if (!colDef.index && !colDef.unique && !this.secondaryIndexes.has(`${tableName}:idx:${column}`)) { throw new DatabaseError(`Index on column "${column}" does not exist in table "${tableName}"`, 'INDEX_NOT_FOUND'); } // v0.7.4: 建表 UNIQUE 约束不可通过 DROP INDEX 解除 —— 此前 colDef.unique = false // 静默解除约束(重启后 persistSchemas 使约束永久消失)。对齐 SQLite 语义: // 约束随建表存在,解除需重建表;仅 CREATE UNIQUE INDEX 添加的约束可随索引删除。 const uniqueKey = `${tableName}:${column}`; if (colDef.unique && !this.uniqueIndexCols.has(uniqueKey)) { throw new DatabaseError( `Cannot drop index on column "${column}" in table "${tableName}": ` + 'UNIQUE constraint defined at table creation must be removed by recreating the table', 'NOT_SUPPORTED', ); } colDef.index = false; colDef.unique = false; this.uniqueIndexCols.delete(uniqueKey); const idxKey = `${tableName}:idx:${column}`; const idxLsm = this.secondaryIndexes.get(idxKey); if (idxLsm) { await idxLsm.clear(); this.secondaryIndexes.delete(idxKey); } await this.persistSchemas(); } // ======================================================================= // 事务 // ======================================================================= async beginTransaction(): Promise { if (this.currentTxnId) throw new DatabaseError('Transaction already in progress', 'TX_ACTIVE'); this.currentTxnId = this.mvcc.beginTransaction(); this.txnSnapshot = new Map(); // v0.8.0: 事务内 WAL 记录计数从 0 开始(保存点边界依赖它) this.txnWalRecordCount = 0; // v0.7.3-fix: WAL BEGIN 写失败回滚内存事务状态 —— 此前 append 抛错(full 模式) // 时 currentTxnId 已设置 → TX_ACTIVE 永久泄漏(后续无法开始新事务)。 // 回滚 mvcc 登记 + 快照后重抛,调用方可重试。 try { await this.wal.append({ type: WALRecordType.BEGIN, txnId: this.currentTxnId, tableName: '', key: '', }); this.txnWalRecordCount = 1; // BEGIN 本身占一条 } catch (error) { this.mvcc.rollbackTransaction(this.currentTxnId); this.currentTxnId = null; this.txnSnapshot = null; throw error; } } async commitTransaction(): Promise { if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE'); // v0.4.3-fix: 先持久化 WAL COMMIT,再合并快照到 LSM — // 崩溃在 WAL 提交后、快照合并前:恢复时 WAL 重放数据,重启一致; // 崩溃在 WAL 提交前:commitTransaction 尚未返回,事务视为未提交(可回滚) await this.wal.append({ type: WALRecordType.COMMIT, txnId: this.currentTxnId, tableName: '', key: '', }); await this.wal.flush(); if (this.txnSnapshot) { for (const [key, value] of this.txnSnapshot) { if ((value as unknown as Record).__txn_deleted) { this.lsm.delete(key); } else { this.lsm.put(key, value); } } } this.mvcc.commitTransaction(this.currentTxnId); // v0.8.0 根治:事务结束时清空 savepoint 表。 // 此前 commit/rollback **都不清理** savepoints,于是: // 1. 上一个事务的名字仍被占用 → 新事务 SAVEPOINT 同名报 "already exists"; // 2. 新事务 ROLLBACK TO 陈旧名字会拿到旧事务快照,把当前事务的写入静默替换。 this.savepoints.clear(); this.savepointWalBoundary.clear(); this.txnWalRecordCount = 0; this.currentTxnId = null; this.txnSnapshot = null; } async rollbackTransaction(): Promise { if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE'); const txnId = this.currentTxnId; // v0.7.3-fix: 先持久化 WAL ROLLBACK,再回滚内存 —— 与 commitTransaction 的 // "WAL 领先内存"(v0.4.3-fix)对齐。此前内存先回滚、ROLLBACK 记录后写: // full 模式写失败时崩溃重放无 ROLLBACK 记录 → 已回滚事务的数据复活。 // 现在写失败 → 内存未回滚、事务仍活跃(调用方可重试),崩溃后重放 // 看到 ROLLBACK 记录同样不会复活数据。 await this.wal.append({ type: WALRecordType.ROLLBACK, txnId, tableName: '', key: '', }); // v0.3.3: 记录事务涉及的表(用于回滚后重建索引,消除索引残留) const affectedTables = new Set(); if (this.txnSnapshot) { for (const key of this.txnSnapshot.keys()) { const idx = key.indexOf(':'); if (idx > 0) affectedTables.add(key.slice(0, idx)); } } this.mvcc.rollbackTransaction(txnId); this.txnSnapshot = null; // v0.8.0:事务结束清空 savepoint(理由同 commitTransaction) this.savepoints.clear(); this.savepointWalBoundary.clear(); this.txnWalRecordCount = 0; this.currentTxnId = null; // v0.3.3: 事务内直接写入了二级索引 LSM,回滚后全量重建受影响表的索引 for (const tableName of affectedTables) { if (this.schemas.has(tableName)) { await this.reindexTable(tableName); } } } // ---- Savepoint 嵌套事务 ---- private savepoints: Map> | null }> = new Map(); /** * v0.8.0: 当前事务在 WAL 中已成功追加的记录条数。 * * 用途:保存点需要记录"回滚后应保留到哪一条",否则恢复时无法区分 * "保存点之前的写入"(应保留)与"保存点之后的写入"(应丢弃)。 */ private txnWalRecordCount = 0; /** 保存点 → 该保存点时事务的 WAL 记录边界 */ private savepointWalBoundary = new Map(); async savepoint(name: string): Promise { if (!this.currentTxnId) throw new DatabaseError('No active transaction for savepoint', 'TX_NONE'); if (this.savepoints.has(name)) throw new DatabaseError(`Savepoint "${name}" already exists`, 'SAVEPOINT_EXISTS'); // 保存当前事务快照 this.savepoints.set(name, { txnId: this.currentTxnId, snapshot: this.txnSnapshot ? new Map(this.txnSnapshot) : null, }); // v0.8.0: 记录边界(该事务已追加的记录条数)—— 见 txnWalRecordCount 说明 this.savepointWalBoundary.set(`${this.currentTxnId}:${name}`, this.txnWalRecordCount); } async rollbackToSavepoint(name: string): Promise { const sp = this.savepoints.get(name); if (!sp) throw new DatabaseError(`Savepoint "${name}" not found`, 'SAVEPOINT_NOT_FOUND'); // v0.8.0 根治:savepoint 归属校验。 // 此前只按名字查找,而 savepoints 在 commit/rollback 时**从不清空** —— // 上一个事务遗留的 savepoint 会让当前事务的写入被旧事务快照静默替换 // (实测:本事务 update 后 ROLLBACK TO 陈旧名 + COMMIT,写入凭空消失)。 if (sp.txnId !== this.currentTxnId) { throw new DatabaseError( `Savepoint "${name}" belongs to a different transaction (stale savepoint)`, 'SAVEPOINT_NOT_FOUND', ); } // v0.8.0: 先写 WAL 标记再改内存(与 commit/rollback 的"WAL 领先内存"一致)—— // 否则崩溃恢复会把该事务在 savepoint 之前写入的记录重新应用,已回滚的行复活。 const boundary = this.savepointWalBoundary.get(`${this.currentTxnId}:${name}`) ?? 0; await this.wal.append({ type: WALRecordType.SAVEPOINT_ROLLBACK, txnId: this.currentTxnId!, tableName: '', key: '', data: { replayFromIndex: boundary }, }); this.txnWalRecordCount++; this.savepointWalBoundary.set(`${this.currentTxnId}:${name}`, boundary); // v0.6.3-fix: 记录当前快照涉及的表(回滚后重建索引)—— 此前事务内直写 // 索引 LSM,savepoint 回滚只还原快照 → savepoint 之后的索引条目残留 const affectedTables = new Set(); if (this.txnSnapshot) { for (const key of this.txnSnapshot.keys()) { const idx = key.indexOf(':'); if (idx > 0) affectedTables.add(key.slice(0, idx)); } } // 恢复到 savepoint 时的快照 this.txnSnapshot = sp.snapshot ? new Map(sp.snapshot) : null; // v0.3.3: 清理该事务在 MVCC 版本链中的全部记录(快照已含正确数据, // 版本链仅作 undo 记录,清空后 commit 时 LSM 写入与快照保持一致) this.mvcc.discardVersions(this.currentTxnId!); // 清除此 savepoint 之后的所有 savepoint let found = false; for (const [k] of this.savepoints) { if (k === name) { found = true; continue; } if (found) this.savepoints.delete(k); } // v0.6.3-fix: 重建受影响表二级索引(消除 savepoint 之后的过期索引条目) for (const tableName of affectedTables) { if (this.schemas.has(tableName)) { await this.reindexTable(tableName); } } } async releaseSavepoint(name: string): Promise { if (!this.savepoints.has(name)) throw new DatabaseError(`Savepoint "${name}" not found`, 'SAVEPOINT_NOT_FOUND'); this.savepoints.delete(name); } // ---- 在线备份 ---- async backup(): Promise[]>> { this.ensureOpen(); const result: Record[]> = {}; for (const tableName of this.schemas.keys()) { result[tableName] = await this.getAllRows(tableName); } return result; } // ======================================================================= // 内部 // ======================================================================= private async getAllRows(tableName: string): Promise[]> { const pkCol = this.tablePKs.get(tableName)!; const prefix = `${tableName}:`; // 预加载范围内涉及的 SSTable,避免 rangeScan 时缓存未命中静默丢数据 await this.lsm.prefetchRange(prefix, `${prefix}\uffff`); const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`); const rows = entries.map(([key, value]) => { // v0.8.0: 深拷贝(此前 `{ ...value }` 只做浅拷贝,嵌套 json 值仍与 LSM // 内部对象共享引用 —— 调用方改 rows[0].nested.a 会改写存储) const row = cloneRow(value); row[pkCol] = key.slice(prefix.length); return row; }); // v0.3.3: 事务内合并未提交快照(update/delete/count/clear 也能看到本事务的写入) return this.mergeTxnSnapshot(tableName, rows); } /** * v0.3.3: 将事务未提交快照的变更合并到行列表(新增/更新/删除标记)。 * 幂等操作:行已是最新时不重复修改。 */ private mergeTxnSnapshot(tableName: string, rows: Record[]): Record[] { if (!this.currentTxnId || !this.txnSnapshot) return rows; const pkCol = this.tablePKs.get(tableName)!; const prefix = `${tableName}:`; for (const [key, value] of this.txnSnapshot) { if (!key.startsWith(prefix)) continue; const pk = key.slice(prefix.length); const del = (value as unknown as Record).__txn_deleted; const idx = rows.findIndex((r) => r[pkCol] === pk); if (del) { if (idx >= 0) rows.splice(idx, 1); } else { const row = { ...value, [pkCol]: pk }; if (idx >= 0) rows[idx] = row; else rows.push(row); } } return rows; } private getPK(schema: TableSchema): string { for (const [name, col] of Object.entries(schema.columns)) { if (col.primaryKey) return name; } return Object.keys(schema.columns)[0]; } private validateRow(schema: TableSchema, row: Record): Record { const validated: Record = {}; for (const [colName, colDef] of Object.entries(schema.columns)) { let value = row[colName]; if (value === undefined && colDef.default !== undefined) value = colDef.default; if (colDef.required && (value === undefined || value === null)) { throw new DatabaseError(`Column "${colName}" is required in table "${schema.name}"`, 'VALIDATION_ERROR'); } // v0.7.4: 主键列强制非空(SQL 语义 PK 隐含 NOT NULL)—— // 此前 null/undefined 主键被 String() 化为 "null"/"undefined" 静默入库 if (colDef.primaryKey && (value === undefined || value === null)) { throw new DatabaseError( `Primary key column "${colName}" in table "${schema.name}" cannot be null or undefined`, 'VALIDATION_ERROR', ); } if (value !== undefined && value !== null) { this.checkType(colName, colDef.type, value, colDef); } if (value !== undefined) validated[colName] = value; } return validated; } private checkType(colName: string, type: string, value: unknown, colDef?: ColumnDef): void { checkFieldType('', colName, type as any, value, colDef); } // ======================================================================= // Schema 持久化 // ======================================================================= private async persistSchemas(): Promise { const data: Record> = {}; for (const [name, schema] of this.schemas) { data[name] = schema.columns; } const json = JSON.stringify(data); const buf = new TextEncoder().encode(json).buffer; await this.backend.write('__aria_schemas', buf); } private async loadSchemas(): Promise { const raw = await this.backend.read('__aria_schemas'); if (!raw) return; try { const json = new TextDecoder().decode(raw); const data = JSON.parse(json) as Record>; for (const [tableName, columns] of Object.entries(data)) { const schema: TableSchema = { name: tableName, columns }; this.schemas.set(tableName, schema); this.tablePKs.set(tableName, this.getPK(schema)); } } catch { // 忽略损坏的 schema 数据 } } // ======================================================================= // SSTableStore 构建 // ======================================================================= /** * 创建命名空间隔离的 SSTableStore。 * * 主 LSM 与每个二级索引 LSM 各持有独立实例: * - 文件 key 前缀隔离(sst_ / sst_idx_${table}_${col}_) * - 元数据 key 隔离(__aria_lsm_meta / __aria_lsm_meta_${ns}) * - id 序列独立(避免 v0.2.4 共享 id 空间导致的文件互相覆盖) */ private createSSTableStore(ns: string): SSTableStore { const filePrefix = ns === 'main' ? 'sst_' : `sst_${ns}_`; const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`; let seq = 0; let seqLoaded = false; // v0.4.5: 页面化物理存储(OPFS 后端默认启用)— SSTable 存为 4KB 页面,BufferPool 缓存 const usePages = this.isPageStorage(); const pageStore = usePages ? new PageSSTableStore(this.fileManager, this.bufferPool) : null; const encodeText = (text: string): ArrayBuffer => { return new TextEncoder().encode(text).buffer; }; const readMetaList = async (): Promise => { const raw = await this.backend.read(META_KEY); if (!raw) return []; try { return JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[]; } catch { return []; } }; return { save: async (id, data) => { if (pageStore) { // 页面化:切页写入 BufferPool 并逐页落盘(save 语义 = 已持久化) await pageStore.save(id, data); return; } let buf: ArrayBuffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer; // 压缩(若启用)— 加密由 EncryptedBackend 在 backend 层透明处理(v0.4.5) if (this.config.compression) { const compressed = compressLZ4(new Uint8Array(buf)); buf = compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer; } await this.backend.write(`${filePrefix}${id}`, buf); }, load: async (id) => { // 页面化读取:meta 有 pageIds → 页面拼接;无(旧数据)→ 整 value if (pageStore) { const metas = await readMetaList(); const meta = metas.find((m) => m.id === id); if (meta && meta.pageIds && meta.pageIds.length > 0) { return pageStore.load(id, meta.pageIds, meta.totalSize); } } const raw = await this.backend.read(`${filePrefix}${id}`); if (!raw) return null; let buf = new Uint8Array(raw); // 解压(若启用)— 解密由 EncryptedBackend 在 backend 层透明处理(v0.4.5) if (this.config.compression) { // v0.4.5: 压缩流自带原始大小头,无需外部估算 buf = decompressLZ4(buf) as Uint8Array; } return buf; }, delete: async (id) => { if (pageStore) { const metas = await readMetaList(); const meta = metas.find((m) => m.id === id); if (meta && meta.pageIds && meta.pageIds.length > 0) { await pageStore.delete(id, meta.pageIds); } } await this.backend.delete(`${filePrefix}${id}`); }, allocateId: async () => { // 从本命名空间的 meta 恢复 id 序列,保证单调递增且不与其他 LSM 冲突 if (!seqLoaded) { const metas = await readMetaList(); seq = metas.reduce((m, x) => Math.max(m, x.id), 0); seqLoaded = true; } return ++seq; }, listMeta: readMetaList, saveMeta: async (meta) => { const list = await readMetaList(); // v0.4.5: 页面化时把页面 ID 列表注入 meta(save 后、saveMeta 前由 LSM 顺序调用) const pageIds = pageStore?.getPageIds(meta.id); const metaWithPages = pageIds && pageIds.length > 0 ? { ...meta, pageIds } : meta; // 更新或添加 const idx = list.findIndex((m) => m.id === meta.id); if (idx >= 0) list[idx] = metaWithPages; else list.push(metaWithPages); await this.backend.write(META_KEY, encodeText(JSON.stringify(list))); }, deleteMeta: async (id) => { const list = await readMetaList(); const filtered = list.filter((m) => m.id !== id); await this.backend.write(META_KEY, encodeText(JSON.stringify(filtered))); }, }; } /** v0.4.5: 是否启用页面化物理存储(默认 OPFS / KVStore 后端启用,显式配置可覆盖) */ private isPageStorage(): boolean { if (this.config.pageStorage === true) return true; if (this.config.pageStorage === false) return false; // v0.6.1: kv 后端同样页面化 — 整 value SSTable(4MB)超过 1MB 页面缓存时 // 每次读取全量重载;拆 4KB 页面后缓存按页命中,大数据量读放大消除 return this.config.storageBackend === 'opfs' || this.config.storageBackend === 'kv'; } // ======================================================================= // WAL 恢复 // ======================================================================= private applyWALRecord(record: WALRecord): void { switch (record.type) { case WALRecordType.INSERT: case WALRecordType.UPDATE: if (record.data) { this.lsm.put(`${record.tableName}:${record.key}`, record.data); } break; case WALRecordType.DELETE: this.lsm.delete(`${record.tableName}:${record.key}`); break; case WALRecordType.CREATE_TABLE: if (record.data?.schema) { try { const s = JSON.parse(record.data.schema as string) as TableSchema; if (!this.schemas.has(s.name)) { this.schemas.set(s.name, s); this.tablePKs.set(s.name, this.getPK(s)); } } catch { /* skip */ } } break; case WALRecordType.COMMIT: case WALRecordType.ROLLBACK: case WALRecordType.BEGIN: break; } } /** * v0.3.3: DROP_TABLE 恢复 — 删除 schema 并清除主 LSM 中该表的所有残留数据。 * * 此前 DROP_TABLE 在恢复时被忽略,而 CREATE_TABLE 回放会重建 schema, * 导致崩溃后"已删除的表和数据复活"(实证 P0 bug)。 */ private async applyDropTableRecovery(tableName: string): Promise { if (!tableName) return; // v0.4.2-fix: 清理该表二级索引(崩溃恢复路径同样不留孤儿索引) await this.cleanupTableIndexes(tableName); this.schemas.delete(tableName); this.tablePKs.delete(tableName); // 清除主 LSM 中该表前缀的所有数据(含 SSTable 中的旧数据) const prefix = `${tableName}:`; const endKey = `${prefix}\uffff`; await this.lsm.prefetchRange(prefix, endKey); const entries = await this.lsm.rangeScan(prefix, endKey); for (const [key] of entries) { this.lsm.delete(key); } } // ======================================================================= // 二级索引 // ======================================================================= /** v0.6.2: 表中有 unique 约束且索引 LSM 已建的列(唯一性检查范围) */ private uniqueColumns(tableName: string, schema: TableSchema): string[] { const cols: string[] = []; for (const [colName, colDef] of Object.entries(schema.columns)) { if (!colDef.unique) continue; if (this.secondaryIndexes.has(`${tableName}:idx:${colName}`)) cols.push(colName); } return cols; } /** * 唯一性检查。 * * v0.8.0: 由 `checkUniqueSync` 改名并改为 async —— 此前命名为 "Sync" 是因为它 * 依赖"批次级 prefetchPrefixRanges 之后索引数据已在缓存中"这一约定。现在 * LSM 读取自洽(未命中即回源),因此这里可以、也必须 await。 * 索引不含 null 条目(null 值不受唯一约束,与 MemoryEngine 语义一致)。 * @param currentPk 当前行主键(更新路径用于排除自身旧索引条目;插入路径无自身条目) */ private async checkUnique( tableName: string, uniqueCols: string[], row: Record, currentPk: string, ): Promise { for (const colName of uniqueCols) { const val = row[colName]; if (val === undefined || val === null) continue; const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`); if (!idxLsm) continue; const prefix = `${String(val)}:`; const entries = await idxLsm.rangeScan(prefix, `${prefix}\uffff`); for (const [, entry] of entries) { const pk = (entry as unknown as { pk?: string }).pk; if (pk !== undefined && pk !== currentPk) { throw new DatabaseError( `Unique constraint violation on column "${colName}" in table "${tableName}"`, 'UNIQUE_VIOLATION', ); } } } } /** 更新行的二级索引条目 */ private updateSecondaryIndexes( tableName: string, pkValue: string, newRow: Record | null, oldRow: Record | null, ): void { const schema = this.schemas.get(tableName); if (!schema) return; for (const [colName, colDef] of Object.entries(schema.columns)) { // v0.3.3: 主键列不建冗余二级索引(主 LSM 即 PK 索引) if (!colDef.index && !colDef.unique) continue; const idxKey = `${tableName}:idx:${colName}`; const idxLsm = this.secondaryIndexes.get(idxKey); if (!idxLsm) continue; // 删除旧值 if (oldRow) { const oldVal = oldRow[colName]; if (oldVal !== undefined && oldVal !== null) { idxLsm.delete(`${String(oldVal)}:${pkValue}`); } } // 插入新值 if (newRow) { const newVal = newRow[colName]; if (newVal !== undefined && newVal !== null) { idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue }); } } } } /** 通过二级索引快速查找 */ private async tryIndexLookup( tableName: string, query: QueryPlan, ): Promise[] | null> { if (!query.where) return null; const schema = this.schemas.get(tableName); if (!schema) return null; const pkCol = this.tablePKs.get(tableName)!; // v0.7.3: 递归展开 $and 中的等值条件 —— 此前仅顶层键, // `WHERE a AND b`(解析为顶层 $and)永远全表扫描,索引形同虚设。 // $or/$not 语义不适用单索引下推,保守跳过。命中索引后 find 仍以 // 全条件 matchWhere 过滤(子集语义安全)。 const flat: [string, unknown][] = []; const collect = (w: WhereCondition): void => { for (const [k, v] of Object.entries(w)) { if (k === '$and') { for (const sub of (v as WhereCondition[])) collect(sub); continue; } if (k === '$or' || k === '$not' || k === '$exists') continue; flat.push([k, v]); } }; collect(query.where); for (const [col, condition] of flat) { const colDef = schema.columns[col]; const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey); if (!hasIndex && col !== pkCol) continue; // PK 等值 → 主 LSM 精确查找 if (col === pkCol) { if (typeof condition !== 'object' || condition === null) { const key = `${tableName}:${condition}`; await this.lsm.prefetchKeys([key]); const value = await this.lsm.get(key); return value ? [{ ...value, [pkCol]: condition }] : []; } const cond = condition as Record; if ('$eq' in cond) { const key = `${tableName}:${cond.$eq}`; await this.lsm.prefetchKeys([key]); const value = await this.lsm.get(key); return value ? [{ ...value, [pkCol]: cond.$eq }] : []; } // v0.3.3: PK $in → 主 LSM 多次精确查找(替代冗余 PK 二级索引) if ('$in' in cond && Array.isArray(cond.$in)) { const keys = cond.$in.map((v) => `${tableName}:${v}`); await this.lsm.prefetchKeys(keys); const rows: Record[] = []; const seen = new Set(); // v0.4.1: IN 子查询可能含重复值,按 pk 去重 for (const v of cond.$in) { const pk = String(v); if (seen.has(pk)) continue; const value = await this.lsm.get(`${tableName}:${pk}`); if (value) { seen.add(pk); rows.push({ ...value, [pkCol]: pk }); } } return rows; } // v0.3.3: PK 范围查询 → 主 LSM 前缀扫描 + 条件过滤(修复字符串算术 bug) if ('$gt' in cond || '$gte' in cond || '$lt' in cond || '$lte' in cond) { const prefix = `${tableName}:`; await this.lsm.prefetchRange(prefix, `${prefix}\uffff`); const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`); const rows: Record[] = []; for (const [key, value] of entries) { const candidate = { ...value, [pkCol]: key.slice(prefix.length) }; if (matchWhere(candidate, { [pkCol]: condition })) rows.push(candidate); } return rows; } } // 二级索引查找 const idxKey = `${tableName}:idx:${col}`; const idxLsm = this.secondaryIndexes.get(idxKey); if (!idxLsm) continue; // $eq → 精确查找 if (typeof condition !== 'object' || condition === null) { // v0.6.2-fix(P1): IS NULL(条件为 null)不走索引 —— 索引不含 null 条目, // String(null)="null" 查找返回空并短路全表 → 索引列 IS NULL 恒空 if (condition === null) continue; return this.indexScanToRows(tableName, pkCol, idxLsm, String(condition), String(condition)); } const c = condition as Record; if ('$eq' in c) { // v0.6.2-fix(P1): 同上,$eq: null(IS NULL)不走索引 if (c.$eq === null) continue; const v = String(c.$eq); return this.indexScanToRows(tableName, pkCol, idxLsm, v, v); } // $in → 多次精确查找 if ('$in' in c && Array.isArray(c.$in)) { // v0.6.2-fix(P1): IN 列表含 null 不走索引(索引不含 null 条目,会漏匹配 null 行) if (c.$in.some((v) => v === null)) continue; // v0.7.3-perf: 批级预加载全部值的索引范围 + 主表行(各一次 drainChain)—— // 此前逐值 indexScanToRows:每个值一次 prefetchRange + prefetchKeys, // 后台 compaction 长耗时时 N 倍放大(与 v0.6.1 修的 insert 批量预加载 // 性能悬崖同类)。批级预加载后循环内同步 rangeScan/get。 const values = c.$in.map((v) => String(v)); await idxLsm.prefetchPrefixRanges(values.map((v): [string, string] => [v, `${v}\uffff`])); const results: Record[] = []; const seenPks = new Set(); // v0.4.1: IN 值可能重复,按 pk 去重 const pks: string[] = []; for (const val of values) { const entries = await idxLsm.rangeScan(val, `${val}\uffff`); for (const [, idxEntry] of entries) { const pk = (idxEntry as { pk?: string }).pk; if (pk && !seenPks.has(pk)) { seenPks.add(pk); pks.push(pk); } } } await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`)); for (const pk of pks) { const row = await this.lsm.get(`${tableName}:${pk}`); if (row) results.push({ ...row, [pkCol]: pk }); } return results; } // $gt / $gte / $lt / $lte → 范围扫描 if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) { // v0.6.2-fix(P1): 此前用 Number(v)±1 构造边界 key —— 小数数值 // ($gt:2 → "3:",漏 2.5)与字符串("NaN:" 前缀错位,数字/大写开头值被漏) // 静默丢数据。改为全索引扫描 + 行级 matchWhere 过滤(与主键范围路径同方案), // 边界语义与 where-matcher 完全一致。 const rows = await this.indexScanToRows(tableName, pkCol, idxLsm, '', '\uffff'); return rows.filter((row) => matchWhere(row, { [col]: condition })); } } return null; } /** 从索引扫描结果恢复完整行 */ private async indexScanToRows( tableName: string, pkCol: string, idxLsm: LSM, startKey: string, endKey: string, ): Promise[]> { // 使用前缀扫描:endKey 需要包含 \uffff 以匹配所有带后缀的 key const actualEndKey = endKey.includes('\uffff') ? endKey : `${endKey}\uffff`; // 预加载索引 LSM 与主 LSM 涉及的 SSTable await idxLsm.prefetchRange(startKey, actualEndKey); const entries = await idxLsm.rangeScan(startKey, actualEndKey); const pks: string[] = []; for (const [, idxEntry] of entries) { const pk = (idxEntry as any).pk as string; if (pk) pks.push(pk); } await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`)); const rows: Record[] = []; for (const pk of pks) { const row = await this.lsm.get(`${tableName}:${pk}`); if (row) rows.push({ ...row, [pkCol]: pk }); } return rows; } // ======================================================================= // 辅助 // ======================================================================= /** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */ private tryGC(): void { this.gcCounter++; if (this.gcCounter >= 10) { this.mvcc.gc(100); this.gcCounter = 0; } } /** 回收主 LSM 与所有二级索引 LSM 的临时缓存超限 */ private trimAllCaches(): void { this.lsm.trimCache(); for (const idxLsm of this.secondaryIndexes.values()) { idxLsm.trimCache(); } } /** 检查内存预算,超出时强制 flush + GC */ private checkMemoryBudget(): void { const maxBytes = this.config.maxMemoryMB * 1024 * 1024; const used = this.lsm.getEstimatedMemory(); if (used > maxBytes) { this.lsm.flush().catch(() => {}); this.mvcc.gc(50); } } /** * ANALYZE: 收集表统计信息 * 返回行数、平均行大小、索引深度等 */ async analyzeTable(tableName: string): Promise> { this.ensureOpen(); this.ensureTable(tableName); const rows = await this.getAllRows(tableName); // v0.7.3: 统计汇总主 LSM + 该表全部二级索引 LSM —— 此前只统计主 LSM, // 表带多个索引时索引深度/SSTable 数量严重低估 let sstableCount = this.lsm.getStats().sstableCount; let memtableSize = this.lsm.getStats().memtableSize; let indexDepth = this.lsm.getStats().levelCounts.filter((c: number) => c > 0).length; for (const [idxKey, idxLsm] of this.secondaryIndexes) { if (!idxKey.startsWith(`${tableName}:idx:`)) continue; const s = idxLsm.getStats(); sstableCount += s.sstableCount; memtableSize += s.memtableSize; indexDepth = Math.max(indexDepth, s.levelCounts.filter((c: number) => c > 0).length); } const stats: Record = { table: tableName, rowCount: rows.length, avgRowSize: rows.length > 0 ? Math.round(rows.reduce((s, r) => s + JSON.stringify(r).length, 0) / rows.length) : 0, indexDepth, sstableCount, memtableSize, estimatedMemory: this.lsm.getEstimatedMemory(), }; // 列基数统计 const schema = this.schemas.get(tableName); if (schema && rows.length > 0) { const columnStats: Record = {}; for (const colName of Object.keys(schema.columns)) { const values = new Set(rows.map((r) => String(r[colName]))); columnStats[colName] = { distinctValues: values.size }; } stats.columnStats = columnStats; } return stats; } /** * REINDEX: 重建指定表的所有二级索引 */ async reindexTable(tableName: string): Promise { this.ensureOpen(); this.ensureTable(tableName); return this.reindexTableInternal(tableName); } /** v0.4.2-fix: 重建索引内部实现(不校验 opened,供 open 恢复流程调用) */ private async reindexTableInternal(tableName: string): Promise { const schema = this.schemas.get(tableName); if (!schema) return 0; let rebuiltCount = 0; // v0.7.4-perf: 单次全表扫描重建全部索引列 —— 此前每个索引列各做一次 // getAllRows(N 列 × M 行全表扫描 + 每次 prefetchRange drainChain), // 多索引大表 REINDEX/崩溃恢复按索引列数线性放大 const pkCol = this.tablePKs.get(tableName)!; const idxCols = Object.entries(schema.columns).filter( ([, colDef]) => colDef.index || colDef.unique, ); if (idxCols.length === 0) return 0; const rows = await this.getAllRows(tableName); for (const [colName] of idxCols) { const idxKey = `${tableName}:idx:${colName}`; const idxLsm = this.secondaryIndexes.get(idxKey); if (!idxLsm) continue; // 清空旧索引 await idxLsm.clear(); rebuiltCount++; // 从主 LSM 重建索引 for (const row of rows) { const val = row[colName]; if (val !== undefined && val !== null) { idxLsm.put(`${String(val)}:${row[pkCol]}`, { pk: row[pkCol] }); } } } return rebuiltCount; } /** * VACUUM: 压缩 LSM + 清理碎片 */ async vacuum(): Promise<{ compactedLevels: number; gcVersions: number }> { this.ensureOpen(); // 强制 flush memtable await this.lsm.flush(); // 压缩各层级 for (let level = 0; level < 6; level++) { if (this.lsm.getStats().levelCounts[level] >= 2) { await this.lsm.compactLevel(level); } } // GC MVCC 版本(保留最新 10 个) const beforeGC = this.mvcc.getGlobalLSN(); this.mvcc.gc(10); return { compactedLevels: 6, gcVersions: beforeGC }; } private ensureOpen(): void { if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN'); } /** v0.4.2-fix: Aria 事务中 DDL 显式拒绝(结构变更无法通过行快照回滚) */ private ensureNoDDLInTransaction(op: string): void { if (this.currentTxnId) { throw new DatabaseError( `${op} is not supported inside a transaction (AriaEngine DDL is not transactional)`, 'NOT_SUPPORTED', ); } } private ensureTable(tableName: string): void { if (!this.schemas.has(tableName)) { throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND'); } } }