thzxx
89243ef6eb
A3 已回滚的行在崩溃重启后复活（静默数据错误）
`ROLLBACK TO <savepoint>` 只改内存快照、**不写 WAL**，而 COMMIT 会把整个 txnId
标记为已提交，恢复时按"该事务的全部记录"重放 → 被回滚掉的写入被重新应用。
实测：事务内插入 a、savepoint、插入 b、回滚到 savepoint、提交 →
实时只剩 a，崩溃重开变成 a+b。
根治：新增 WALRecordType.SAVEPOINT_ROLLBACK（走既有 data 字段携带
{ replayFromIndex } 边界，二进制格式不变、旧库记录仍可解析）。
- 写入侧：savepoint() 记录"该事务当时已追加的记录条数"；rollbackToSavepoint()
先写标记再改内存（与 commit/rollback 的"WAL 领先内存"一致）。
- 恢复侧：按**事务内**下标计算窗口 —— 保留 [0, keepUpTo)，丢弃
[keepUpTo, 最后一个标记)，标记之后的记录照常保留。
注意不能拿全局下标比较：全局数组里混有 txnId=0 的非事务记录（CREATE_TABLE
等）与其它事务的记录。这个差一错误在实现过程中被测试抓出并修正。
- 事务内 WAL 记录计数（txnWalRecordCount）在 5 个 appendBatch 站点与 BEGIN
处维护，事务开始/结束时归零。
A4 跨事务复用的陈旧 savepoint 静默丢弃当前事务的写入
savepoints 在 commit/rollback 时**从不清空**，且 rollbackToSavepoint 不校验归属。
实测：上一个事务遗留 savepoint 名 → 新事务 update 后 ROLLBACK TO 该名 + COMMIT，
写入凭空消失（v=5 被回退成 v=9）。
根治：事务结束清空 savepoints 与边界表；rollbackToSavepoint 校验
sp.txnId === currentTxnId，陈旧保存点抛 SAVEPOINT_NOT_FOUND。
新增 tests/v080-savepoint.test.ts（4 个用例）：包含"崩溃重放一致性"、
"普通事务不得误伤"、以及"多个保存点回到最早"的语义护栏。
2401 lines
97 KiB
TypeScript
2401 lines
97 KiB
TypeScript
import { cloneRow } from '../interface';
|
||
/**
|
||
* AriaEngine — 自研页面式存储引擎主类
|
||
* @module engine/aria/index
|
||
*
|
||
* v0.4.1: 外键级联 + ALTER TABLE 重写 + clearAll 重置 + 崩溃恢复加固
|
||
*/
|
||
|
||
import type { IStorageEngine } from '../interface';
|
||
import type { QueryPlan, TableSchema, ColumnDef, WhereCondition } from '../../constants';
|
||
import { DatabaseError } from '../../constants';
|
||
import { matchWhere, applyOrderBy, projectColumns, containsUnresolvedSubqueries } from '../../query/where-matcher';
|
||
import { checkFieldType, stripUndefinedUpdates } from '../../table/schema';
|
||
|
||
import type { AriaEngineConfig, SSTableMeta } from './types';
|
||
import { DEFAULT_ARIA_CONFIG } from './types';
|
||
|
||
import { LSM } from './index/lsm';
|
||
import type { SSTableStore } from './index/lsm';
|
||
import { WAL } from './wal/log';
|
||
import { SegmentedWALStore } from './wal/segmented_store';
|
||
import { DatabaseLock } from './locks';
|
||
import { WALRecordType, type WALRecord } from './types';
|
||
import { CheckpointManager } from './wal/checkpoint';
|
||
import { MemoryBackend, type IStorageBackend } from './store/backend';
|
||
import { KVStoreBackend } from './store/kvstore_backend';
|
||
import { OPFSBackend } from './store/opfs_backend';
|
||
import { EncryptedBackend } from './store/encrypted_backend';
|
||
import { PageSSTableStore } from './store/page_sstable_store';
|
||
import { FileManager } from './store/file_manager';
|
||
import { MVCCManager } from './transaction/mvcc';
|
||
import { BufferPool } from './buffer/pool';
|
||
import { compressLZ4, decompressLZ4 } from './compression/lz4';
|
||
import type { WALRecord as _WALRecord } from './types';
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// AriaEngine
|
||
// ---------------------------------------------------------------------------
|
||
|
||
export class AriaEngine implements IStorageEngine {
|
||
readonly name = 'aria';
|
||
|
||
private config!: Required<Omit<AriaEngineConfig, 'encryption' | 'pageStorage'>> &
|
||
Pick<AriaEngineConfig, 'encryption' | 'pageStorage'>;
|
||
private lsm!: LSM; // 主键索引 LSM
|
||
private wal!: WAL;
|
||
private checkpointManager!: CheckpointManager;
|
||
private backend!: IStorageBackend;
|
||
private opened = false;
|
||
private dbName = '';
|
||
|
||
// v0.4.5: 页面化物理存储（FileManager + BufferPool 提升为引擎字段，close/repair 时落盘/清理）
|
||
private fileManager!: FileManager;
|
||
private bufferPool!: BufferPool;
|
||
|
||
// v0.4.5: 多标签页独占锁（Web Locks API，OPFS 等无事务后端防并发写）
|
||
private dbLock: DatabaseLock | null = null;
|
||
|
||
// 表结构
|
||
private schemas: Map<string, TableSchema> = new Map();
|
||
private tablePKs: Map<string, string> = new Map();
|
||
private opCounter = 0;
|
||
|
||
// 二级索引：table.colKey → LSM
|
||
private secondaryIndexes: Map<string, LSM> = new Map();
|
||
|
||
/**
|
||
* v0.7.4: 由 CREATE UNIQUE INDEX 添加的 unique 列（table:col）。
|
||
* 与建表 UNIQUE 约束区分：DROP INDEX 只允许解除索引来源的 unique，
|
||
* 建表约束需重建表（对齐 SQLite 语义，此前静默解除且重启后永久消失）。
|
||
* 注：重启后无法区分历史来源，schema 中的 unique 一律按建表约束保护（保守）。
|
||
*/
|
||
private uniqueIndexCols: Set<string> = new Set();
|
||
|
||
// MVCC 事务
|
||
private mvcc: MVCCManager = new MVCCManager();
|
||
private currentTxnId: number | null = null;
|
||
private txnSnapshot: Map<string, Record<string, unknown>> | null = null;
|
||
private gcCounter = 0;
|
||
|
||
constructor(config: AriaEngineConfig = {}) {
|
||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||
}
|
||
|
||
// =======================================================================
|
||
// 生命周期
|
||
// =======================================================================
|
||
|
||
async open(dbName: string, _version: number): Promise<void> {
|
||
if (this.opened) return;
|
||
// v0.4.2-fix: 引擎内部错误统一包装为 DatabaseError（ARIA_OPEN_ERROR），
|
||
// 应用层可拿到 code 分类处理，不再抛出原生 RangeError/TypeError
|
||
try {
|
||
await this.openInternal(dbName);
|
||
} catch (error) {
|
||
// 打开失败：释放已获取的锁（避免锁泄漏阻塞其他标签页）
|
||
if (this.dbLock) {
|
||
try { await this.dbLock.release(); } catch { /* ignore */ }
|
||
this.dbLock = null;
|
||
}
|
||
if (error instanceof DatabaseError) throw error;
|
||
throw new DatabaseError(
|
||
`Failed to open AriaEngine database "${dbName}"`,
|
||
'ARIA_OPEN_ERROR',
|
||
error,
|
||
);
|
||
}
|
||
}
|
||
|
||
/** open 内部实现（错误包装在 open 外层） */
|
||
private async openInternal(dbName: string): Promise<void> {
|
||
this.dbName = dbName;
|
||
|
||
// v0.4.5: 多标签页独占锁（Web Locks）— 不支持的环境降级为无锁（文档注明）
|
||
const lock = new DatabaseLock();
|
||
this.dbLock = lock;
|
||
const lockAcquired = await lock.acquire(dbName);
|
||
if (!lockAcquired) {
|
||
// eslint-disable-next-line no-console
|
||
console.warn(
|
||
`[AriaEngine] Web Locks API unavailable: no multi-tab protection for "${dbName}" ` +
|
||
'(open the same database in multiple tabs may corrupt data)',
|
||
);
|
||
}
|
||
|
||
// 1. 存储后端（可选全库加密包装）
|
||
let baseBackend: IStorageBackend;
|
||
if (this.config.storageBackend === 'opfs') {
|
||
baseBackend = new OPFSBackend();
|
||
} else if (this.config.storageBackend === 'kv') {
|
||
// v0.6.1: 自研 KVStore 后端（aria 完全跑在自研存储栈上，不依赖浏览器 OPFS）
|
||
baseBackend = new KVStoreBackend();
|
||
} else {
|
||
baseBackend = new MemoryBackend();
|
||
}
|
||
await baseBackend.open(dbName);
|
||
|
||
// v0.4.5: encryption 配置 → 透明加密封装（密码错误/数据损坏在 open 或首次读取时暴露）
|
||
if (this.config.encryption?.password) {
|
||
this.backend = new EncryptedBackend(baseBackend, this.config.encryption.password);
|
||
} else {
|
||
this.backend = baseBackend;
|
||
// 反向检测：库中存在密钥元数据但未提供密码 → 拒绝打开（避免密文被当明文解析成空库）
|
||
if (await baseBackend.exists('__aria_keymeta')) {
|
||
await baseBackend.close();
|
||
throw new DatabaseError(
|
||
'Database is encrypted: provide encryption.password to open it',
|
||
'ARIA_ENCRYPT_REQUIRED',
|
||
);
|
||
}
|
||
}
|
||
await this.backend.open(dbName);
|
||
|
||
// 2a. FileManager (PageIO 实现) + Buffer Pool
|
||
this.fileManager = new FileManager(this.backend);
|
||
await this.fileManager.init(dbName);
|
||
this.bufferPool = new BufferPool(this.fileManager, this.config.bufferPoolPages);
|
||
|
||
// 2. 构建 SSTableStore
|
||
const sstableStore = this.createSSTableStore('main');
|
||
|
||
// 3. 初始化主 LSM（PK 索引）
|
||
this.lsm = new LSM({
|
||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||
blockSize: this.config.pageSize,
|
||
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||
// SSTable 缓存上限 = BufferPool 页数 × 页面大小（默认 256 页 ≈ 1MB 可控内存）
|
||
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
|
||
sstableStore,
|
||
});
|
||
|
||
// 4. 初始化 WAL
|
||
// v0.4.5: 分片式 WAL 存储（__wal_%06d.bin），序号内嵌记录字节流无需 count 键，
|
||
// append 单文件原子写；空洞检测截断；兼容旧格式 __wal_N + __wal_count
|
||
this.wal = new WAL(
|
||
new SegmentedWALStore(this.backend),
|
||
this.config.walEnabled,
|
||
this.config.walSyncMode,
|
||
);
|
||
|
||
// 5. 恢复 Schema
|
||
await this.loadSchemas();
|
||
|
||
// v0.4.2-fix: 为 schema 中带 index/unique 标记的列重建二级索引 LSM。
|
||
// 此前重开只恢复 schema 不恢复索引 LSM → 索引查询静默回退全表、
|
||
// createIndex 因 colDef 已有标记直接 return → 索引永久缺失。
|
||
// 索引数据已持久化在独立命名空间（sst_idx_* / meta），init() 直接加载。
|
||
for (const [tableName, schema] of this.schemas) {
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||
if ((colDef.index || colDef.unique) && colName !== pkCol) {
|
||
const idxKey = `${tableName}:idx:${colName}`;
|
||
if (!this.secondaryIndexes.has(idxKey)) {
|
||
const idxLsm = new LSM({
|
||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||
blockSize: this.config.pageSize,
|
||
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
|
||
sstableStore: this.createSSTableStore(`idx_${tableName}_${colName}`),
|
||
});
|
||
await idxLsm.init();
|
||
this.secondaryIndexes.set(idxKey, idxLsm);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 6. 初始化 LSM（加载 SSTable 元数据）
|
||
await this.lsm.init();
|
||
|
||
// 7. WAL 恢复（两阶段：先扫描事务边界，仅回放已提交事务）
|
||
const committedTxns = new Set<number>();
|
||
const allRecords: WALRecord[] = [];
|
||
await this.wal.recover((r) => allRecords.push(r));
|
||
// 第一遍：确定已提交事务
|
||
for (const r of allRecords) {
|
||
if (r.type === WALRecordType.COMMIT) committedTxns.add(r.txnId);
|
||
if (r.type === WALRecordType.ROLLBACK) committedTxns.delete(r.txnId);
|
||
}
|
||
|
||
// v0.8.0 根治：确定每个事务的**回放起始下标**。
|
||
//
|
||
// `ROLLBACK TO <savepoint>` 只回滚内存快照，日志里仍留有 savepoint 之前
|
||
// 写入的记录；COMMIT 又把整个 txnId 标记为已提交 —— 于是那些被回滚掉的写入
|
||
// 在重启时被重新应用，**已回滚的行复活**（实测：实时只剩 a，崩溃重开变成 a+b）。
|
||
//
|
||
// 现在 savepoint 回滚会写入 SAVEPOINT_ROLLBACK 记录；恢复时该事务只应用
|
||
// **最后一条** SAVEPOINT_ROLLBACK 之后的记录 —— 等价于"回到该保存点"，
|
||
// 与实时态严格一致（这样也就不需要 undo 崩溃前已落盘的旧值）。
|
||
// 每个事务：{ 保留起点, 丢弃终点 } —— 区间 [start, end) 保留。
|
||
//
|
||
// 关键：`replayFromIndex` 是**事务内**的记录序号（写入方按事务计数），
|
||
// 因此这里必须用"该事务的第几条记录"来比较，不能直接拿全局下标 ——
|
||
// 全局下标里还混着 txnId=0 的非事务记录（CREATE_TABLE 等）以及其它事务。
|
||
const globalIndexToTxnIndex = new Map<number, number>();
|
||
const txnRecordCount = new Map<number, number>();
|
||
for (let i = 0; i < allRecords.length; i++) {
|
||
const r = allRecords[i];
|
||
if (r.txnId === 0) continue;
|
||
const n = txnRecordCount.get(r.txnId) ?? 0;
|
||
globalIndexToTxnIndex.set(i, n);
|
||
txnRecordCount.set(r.txnId, n + 1);
|
||
}
|
||
|
||
// 边界语义（必须显式定义，否则差一错误就在这里）：
|
||
// 写入侧记录的 `replayFromIndex = N` 表示"保存点建立时，本事务已成功追加了 N 条记录"，
|
||
// 即事务的第 0..N-1 条记录必须保留（BEGIN 是第 0 条）。
|
||
// 因此恢复侧应保留的**事务内下标区间**是 [0, N)，丢弃 [N, 标记位置)。
|
||
// 等价地：只丢弃"事务内下标 >= N"且"在最后一个标记之前"的记录。
|
||
const txnReplayWindow = new Map<number, { keepUpTo: number; dropFrom: number }>();
|
||
for (let i = 0; i < allRecords.length; i++) {
|
||
const r = allRecords[i];
|
||
if (r.type !== WALRecordType.SAVEPOINT_ROLLBACK) continue;
|
||
const declared = (r.data as { replayFromIndex?: number } | undefined)?.replayFromIndex;
|
||
const keepUpTo = typeof declared === 'number' ? declared : 0;
|
||
const txnIdx = globalIndexToTxnIndex.get(i) ?? 0;
|
||
const prev = txnReplayWindow.get(r.txnId);
|
||
// 多个保存点回滚：保留上界取**最早**的（回到最早的保存点），
|
||
// 丢弃起点取**最后一个**标记的事务内下标。
|
||
txnReplayWindow.set(r.txnId, {
|
||
keepUpTo: prev ? Math.min(prev.keepUpTo, keepUpTo) : keepUpTo,
|
||
dropFrom: txnIdx,
|
||
});
|
||
}
|
||
|
||
// 第二遍：仅应用 txnId==0（非事务）或已提交事务的数据
|
||
for (let i = 0; i < allRecords.length; i++) {
|
||
const r = allRecords[i];
|
||
if (r.txnId === 0 || committedTxns.has(r.txnId)) {
|
||
if (r.type === WALRecordType.SAVEPOINT_ROLLBACK) continue; // 标记记录，无数据
|
||
// 事务内落在 [start, end) 之外的记录：属于被 savepoint 回滚掉的部分，丢弃
|
||
const win = txnReplayWindow.get(r.txnId);
|
||
if (win !== undefined) {
|
||
const txnIdx = globalIndexToTxnIndex.get(i) ?? 0;
|
||
// 保留 [0, keepUpTo)；丢弃 [keepUpTo, dropFrom)；标记之后的记录照常保留
|
||
if (txnIdx >= win.keepUpTo && txnIdx < win.dropFrom) continue;
|
||
}
|
||
if (r.type === WALRecordType.DROP_TABLE) {
|
||
// v0.3.3: DROP_TABLE 回放（异步：需预加载 SSTable 后清除残留数据）
|
||
await this.applyDropTableRecovery(r.tableName);
|
||
} else {
|
||
this.applyWALRecord(r);
|
||
}
|
||
}
|
||
}
|
||
|
||
// v0.3.3: 恢复完成后将回放数据落盘并截断 WAL，
|
||
// 避免每次重启重复回放 + WAL 无限膨胀
|
||
if (allRecords.length > 0) {
|
||
await this.lsm.flush();
|
||
await this.wal.checkpoint();
|
||
// v0.4.2-fix: WAL 回放只更新主 LSM，二级索引 LSM 未同步 →
|
||
// 崩溃前最后一批写入的索引缺失，重开时索引查询丢行。
|
||
// 恢复后全量重建所有表的二级索引（幂等）。
|
||
for (const tableName of this.schemas.keys()) {
|
||
await this.reindexTableInternal(tableName);
|
||
}
|
||
}
|
||
|
||
// 8. Checkpoint Manager（接入 WAL 大小阈值）
|
||
// v0.4.2-fix: 事务活跃时 checkpoint 不得截断 WAL —
|
||
// 否则 BEGIN/INSERT 记录被截断，COMMIT 后崩溃恢复丢失整个事务数据
|
||
this.checkpointManager = new CheckpointManager(
|
||
this.lsm,
|
||
{
|
||
checkpoint: async () => {
|
||
if (this.currentTxnId) return;
|
||
await this.wal.checkpoint();
|
||
},
|
||
flush: async () => {
|
||
if (this.currentTxnId) return;
|
||
await this.wal.flush();
|
||
},
|
||
getBufferedBytes: () => this.wal.getBufferedBytes(),
|
||
getBufferedCount: () => this.wal.getBufferedCount(),
|
||
} as unknown as WAL,
|
||
{
|
||
flushAll: async () => {
|
||
// v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM —
|
||
// 此前只 flush 主 LSM，checkpoint 截断 WAL 后崩溃时索引 memtable 未落盘、
|
||
// WAL 为空跳过重建 → 二级索引静默丢失最后一批条目（生产数据一致性问题）
|
||
await this.lsm.flush();
|
||
for (const idxLsm of this.secondaryIndexes.values()) {
|
||
await idxLsm.flush();
|
||
}
|
||
},
|
||
} as any,
|
||
this.config.checkpointInterval,
|
||
this.config.walSizeThreshold,
|
||
);
|
||
|
||
this.opened = true;
|
||
}
|
||
|
||
async close(): Promise<void> {
|
||
if (!this.opened) return;
|
||
await this.persistSchemas();
|
||
await this.lsm.flush();
|
||
// v0.4.2-fix: 同步落盘全部二级索引 LSM — 此前只 flush 主 LSM，
|
||
// 优雅关闭后索引 memtable 未落盘 → 重开索引为空 → 索引查询返回空结果
|
||
for (const idxLsm of this.secondaryIndexes.values()) {
|
||
await idxLsm.flush();
|
||
}
|
||
// v0.4.5: 页面化存储 — 落盘全部脏页（save 已逐页落盘，此处兜底）
|
||
await this.bufferPool.flushAll();
|
||
await this.wal.flush();
|
||
// v0.4.2-fix: close 前 checkpoint（截断 WAL）—
|
||
// 此前只 flush 不截断，下次打开会重放全部历史 WAL 记录（含已落盘 SSTable 的数据），
|
||
// 重复解析/重复 put 拖慢启动，并与恢复后 flush+checkpoint 竞争放大丢数据
|
||
await this.wal.checkpoint();
|
||
await this.backend.close();
|
||
// v0.4.5: 释放多标签页独占锁（等待锁真正归还）
|
||
if (this.dbLock) {
|
||
await this.dbLock.release();
|
||
this.dbLock = null;
|
||
}
|
||
// v0.4.2-fix: 清空运行期状态（此前 close 后 mvcc/txn 残留，
|
||
// 重开时 beginTransaction 报 TX_ACTIVE 或读到陈旧快照）
|
||
this.schemas.clear();
|
||
this.tablePKs.clear();
|
||
this.secondaryIndexes.clear();
|
||
this.uniqueIndexCols.clear();
|
||
this.mvcc = new MVCCManager();
|
||
this.currentTxnId = null;
|
||
this.txnSnapshot = null;
|
||
this.savepoints.clear();
|
||
this.opCounter = 0;
|
||
this.opened = false;
|
||
}
|
||
|
||
/**
|
||
* v0.4.2-fix: 崩溃恢复/自愈 — 校验并移除损坏 SSTable、截断 WAL、重建二级索引。
|
||
* v0.4.5 增强：清理 OPFS 残留临时文件、清理孤儿页面（meta 未引用的 pg_ 文件）。
|
||
* 应用层检测到异常后调用，无需删库重建。
|
||
*/
|
||
async repair(): Promise<void> {
|
||
this.ensureOpen();
|
||
// v0.6.0-fix: 先清页面缓存再校验 — 缓存中的"完好页面"会掩盖磁盘损坏
|
||
await this.bufferPool.clear();
|
||
// 1. 校验全部 SSTable，移除残缺项（打开时已做一次，此处兜底运行期损坏）
|
||
const removed = await this.lsm.validateAll();
|
||
// 2. 将 WAL 残留数据落盘并截断，避免无限重放（含空洞截断落地）
|
||
await this.lsm.flush();
|
||
await this.wal.checkpoint();
|
||
// 3. 重建所有表的二级索引（修复索引与主数据不一致）
|
||
for (const tableName of this.schemas.keys()) {
|
||
await this.reindexTable(tableName);
|
||
}
|
||
// v0.4.5: 4. 清理 OPFS 残留临时文件（.crswap/.tmp）
|
||
const backendAny = this.backend as unknown as { cleanupStaleFiles?: () => Promise<void> };
|
||
if (typeof backendAny.cleanupStaleFiles === 'function') {
|
||
try { await backendAny.cleanupStaleFiles(); } catch { /* 清理失败不阻塞 */ }
|
||
}
|
||
// v0.4.5: 5. 清理孤儿页面（所有 LSM 命名空间 meta 均未引用的 pg_ 文件）
|
||
await this.cleanupOrphanPages();
|
||
if (removed > 0) {
|
||
// eslint-disable-next-line no-console
|
||
console.warn(`[AriaEngine] repair: removed ${removed} corrupted SSTable(s)`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.4.5: 清理孤儿页面 — 扫描全部 pg_* 文件，未被任何 LSM 命名空间 meta 引用的删除。
|
||
* 孤儿页面来自：崩溃中断的 compaction/删除流程（旧 SSTable 页面残留）。
|
||
*/
|
||
private async cleanupOrphanPages(): Promise<void> {
|
||
const keys = await this.backend.listKeys();
|
||
const pgKeys = keys.filter((k) => /^pg_\d+$/.test(k));
|
||
if (pgKeys.length === 0) return;
|
||
|
||
const used = new Set<number>();
|
||
const collectMeta = async (ns: string): Promise<void> => {
|
||
const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`;
|
||
const raw = await this.backend.read(META_KEY);
|
||
if (!raw) return;
|
||
try {
|
||
const metas = JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[];
|
||
for (const m of metas) {
|
||
if (m.pageIds) {
|
||
for (const pid of m.pageIds) used.add(pid);
|
||
}
|
||
}
|
||
} catch { /* 损坏的 meta 忽略（validateAll 已处理） */ }
|
||
};
|
||
|
||
await collectMeta('main');
|
||
// 收集全部二级索引命名空间
|
||
for (const [tableName, schema] of this.schemas) {
|
||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||
if (colDef.index || colDef.unique) {
|
||
await collectMeta(`idx_${tableName}_${colName}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
const orphanIds = pgKeys
|
||
.map((k) => Number(k.slice('pg_'.length)))
|
||
.filter((pid) => !used.has(pid));
|
||
if (orphanIds.length > 0) {
|
||
await this.backend.deleteMany(orphanIds.map((pid) => `pg_${pid}`));
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.4.1: 重置数据库 — 清空全部数据与表结构（演示页刷新/重新初始化用）。
|
||
* 清空存储后端、LSM、WAL、MVCC 与二级索引，后续可继续使用本实例。
|
||
*/
|
||
async clearAll(): Promise<void> {
|
||
this.ensureOpen();
|
||
// 清空存储后端（页面文件 / WAL 记录 / schema 记录 / 元数据）
|
||
await this.backend.clear();
|
||
// v0.4.5: 清空页面缓存与页面 ID 分配状态
|
||
await this.bufferPool.clear();
|
||
await this.fileManager.clearAll();
|
||
this.schemas.clear();
|
||
this.tablePKs.clear();
|
||
this.secondaryIndexes.clear();
|
||
this.uniqueIndexCols.clear();
|
||
this.lsm.clear();
|
||
this.mvcc = new MVCCManager();
|
||
this.currentTxnId = null;
|
||
this.txnSnapshot = null;
|
||
this.savepoints.clear();
|
||
this.opCounter = 0;
|
||
// 持久化空 schema（防止旧 schema 记录残留）
|
||
await this.persistSchemas();
|
||
// 重置 WAL 状态（backend.clear 已清记录，同步内存计数）
|
||
await this.wal.checkpoint();
|
||
}
|
||
|
||
isOpen(): boolean { return this.opened; }
|
||
|
||
// ---- v0.4.2-fix: 库内元数据（迁移版本持久化用） ----
|
||
|
||
async getMeta(key: string): Promise<string | null> {
|
||
const raw = await this.backend.read(`__meta_${key}`);
|
||
return raw ? new TextDecoder().decode(raw) : null;
|
||
}
|
||
|
||
async setMeta(key: string, value: string): Promise<void> {
|
||
await this.backend.write(`__meta_${key}`, new TextEncoder().encode(value).buffer);
|
||
}
|
||
|
||
// =======================================================================
|
||
// 表管理
|
||
// =======================================================================
|
||
|
||
async createTable(schema: TableSchema): Promise<void> {
|
||
this.ensureOpen();
|
||
// v0.4.2-fix: Aria 事务中 DDL 显式拒绝（事务快照只覆盖行数据，
|
||
// 结构变更无法回滚；Memory/IndexedDB 引擎快照可回滚，行为不一致 → 明确报错而非静默）
|
||
this.ensureNoDDLInTransaction('CREATE TABLE');
|
||
if (this.schemas.has(schema.name)) {
|
||
throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS');
|
||
}
|
||
|
||
this.schemas.set(schema.name, schema);
|
||
this.tablePKs.set(schema.name, this.getPK(schema));
|
||
|
||
// 为索引列创建二级索引 LSM（每个索引使用独立命名空间的 SSTableStore，避免 id/meta 冲突）
|
||
// v0.3.3: 主键列不建冗余二级索引（主 LSM 本身就是 PK 索引，范围查询走前缀扫描）
|
||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||
if (colDef.index || colDef.unique) {
|
||
const idxKey = `${schema.name}:idx:${colName}`;
|
||
if (!this.secondaryIndexes.has(idxKey)) {
|
||
const idxLsm = new LSM({
|
||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||
blockSize: this.config.pageSize,
|
||
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
|
||
sstableStore: this.createSSTableStore(`idx_${schema.name}_${colName}`),
|
||
});
|
||
await idxLsm.init();
|
||
this.secondaryIndexes.set(idxKey, idxLsm);
|
||
}
|
||
}
|
||
}
|
||
|
||
await this.persistSchemas();
|
||
|
||
await this.wal.append({
|
||
type: WALRecordType.CREATE_TABLE,
|
||
txnId: 0,
|
||
tableName: schema.name,
|
||
key: '',
|
||
data: { schema: JSON.stringify(schema) } as unknown as Record<string, unknown>,
|
||
});
|
||
}
|
||
|
||
async dropTable(tableName: string): Promise<void> {
|
||
this.ensureOpen();
|
||
this.ensureNoDDLInTransaction('DROP TABLE');
|
||
this.ensureTable(tableName);
|
||
|
||
// 删除表中所有行
|
||
const rows = await this.getAllRows(tableName);
|
||
for (const row of rows) {
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
this.lsm.delete(`${tableName}:${row[pkCol]}`);
|
||
}
|
||
|
||
// v0.4.2-fix: 清理该表的全部二级索引 LSM 与持久化文件 —
|
||
// 此前残留孤儿索引，重建同名表后旧索引数据污染新表（索引查询返回错误行）
|
||
await this.cleanupTableIndexes(tableName);
|
||
// v0.7.4: 清理该表的 unique 索引来源标记
|
||
const uPrefix = `${tableName}:`;
|
||
for (const k of this.uniqueIndexCols) {
|
||
if (k.startsWith(uPrefix)) this.uniqueIndexCols.delete(k);
|
||
}
|
||
|
||
this.schemas.delete(tableName);
|
||
this.tablePKs.delete(tableName);
|
||
await this.persistSchemas();
|
||
|
||
await this.wal.append({
|
||
type: WALRecordType.DROP_TABLE,
|
||
txnId: 0,
|
||
tableName,
|
||
key: '',
|
||
});
|
||
}
|
||
|
||
/**
|
||
* v0.4.2-fix: 清理指定表的全部二级索引 LSM（内存 + 存储文件 + meta）。
|
||
* dropTable / DROP_TABLE 恢复 / alterTable DROP 索引列 共用。
|
||
*/
|
||
private async cleanupTableIndexes(tableName: string): Promise<void> {
|
||
const prefix = `${tableName}:idx:`;
|
||
const toDelete: string[] = [];
|
||
for (const [idxKey, idxLsm] of this.secondaryIndexes) {
|
||
if (!idxKey.startsWith(prefix)) continue;
|
||
toDelete.push(idxKey);
|
||
try {
|
||
await idxLsm.clear();
|
||
} catch { /* 清理失败不阻塞 */ }
|
||
}
|
||
for (const idxKey of toDelete) {
|
||
this.secondaryIndexes.delete(idxKey);
|
||
}
|
||
}
|
||
|
||
async hasTable(tableName: string): Promise<boolean> {
|
||
// v0.7.1: 未 open 防护统一（此前与 KVStoreEngine 不一致：返回空而非报错）
|
||
this.ensureOpen();
|
||
return this.schemas.has(tableName);
|
||
}
|
||
|
||
async getTableNames(): Promise<string[]> {
|
||
this.ensureOpen();
|
||
return Array.from(this.schemas.keys());
|
||
}
|
||
|
||
async getTableSchema(tableName: string): Promise<TableSchema | null> {
|
||
this.ensureOpen();
|
||
return this.schemas.get(tableName) ?? null;
|
||
}
|
||
|
||
// =======================================================================
|
||
// CRUD
|
||
// =======================================================================
|
||
|
||
async insert(tableName: string, rows: Record<string, unknown>[]): Promise<string[]> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
|
||
const schema = this.schemas.get(tableName)!;
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const pks: string[] = [];
|
||
// v0.3.1: 批量 WAL 写入（组提交），一次 insert 合并为一次落盘
|
||
const walRecords: Omit<WALRecord, 'lsn' | 'checksum'>[] = [];
|
||
|
||
// v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable（一次 drainChain）。
|
||
// 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链，
|
||
// 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖
|
||
// （10 万行 kv 后端从 5ms/批暴跌到 8~11s/批）。批内新数据在 memtable
|
||
// 或 flush 产物（自动入缓存），循环内 lsm.get 始终完整。
|
||
//
|
||
// v0.6.2: 整批预校验（验证失败整批不落库，语义更原子）+ 唯一约束检查。
|
||
const uniqueCols = this.uniqueColumns(tableName, schema);
|
||
const validatedRows: { row: Record<string, unknown>; pkValue: string; key: string }[] = [];
|
||
for (const row of rows) {
|
||
const validated = this.validateRow(schema, row);
|
||
const pkValue = String(validated[pkCol]);
|
||
validatedRows.push({ row: validated, pkValue, key: `${tableName}:${pkValue}` });
|
||
}
|
||
|
||
await this.lsm.prefetchKeys(validatedRows.map((v) => v.key));
|
||
// v0.7.3: 主键批内互查 + 预检 —— 此前 PK 重复检查在写入循环内：
|
||
// 第 N 行重复抛错时，前 N-1 行已 put LSM 且其 WAL 记录随 appendBatch 一起
|
||
// 丢失 → 语句级部分提交 + 内存/WAL 不一致（与 v0.6.2 的 unique 预检同一阶段）。
|
||
const pkSet = new Set<string>();
|
||
for (const { pkValue, key } of validatedRows) {
|
||
if (pkSet.has(pkValue)) {
|
||
throw new DatabaseError(
|
||
`Duplicate primary key "${pkValue}" in table "${tableName}"`,
|
||
'DUPLICATE_KEY',
|
||
);
|
||
}
|
||
pkSet.add(pkValue);
|
||
// v0.8.0: 事务快照优先，否则回源 LSM（lsm.get 现为 async）
|
||
const existing = this.currentTxnId
|
||
? (this.txnSnapshot?.get(key) ?? await this.lsm.get(key))
|
||
: await this.lsm.get(key);
|
||
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
|
||
throw new DatabaseError(
|
||
`Duplicate primary key "${pkValue}" in table "${tableName}"`,
|
||
'DUPLICATE_KEY',
|
||
);
|
||
}
|
||
}
|
||
// v0.6.2: 唯一约束 — 批量预加载本批唯一列涉及的索引范围（一次 drainChain）
|
||
for (const colName of uniqueCols) {
|
||
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`)!;
|
||
await idxLsm.prefetchPrefixRanges(
|
||
validatedRows
|
||
.map((v): [string, string] | null => {
|
||
const val = v.row[colName];
|
||
if (val === undefined || val === null) return null;
|
||
const p = `${String(val)}:`;
|
||
return [p, `${p}\uffff`];
|
||
})
|
||
.filter((r): r is [string, string] => r !== null),
|
||
);
|
||
}
|
||
// v0.6.2: 唯一性整批预检（批内互查 + 索引查）—— 失败整批不落库（原子语义）
|
||
const batchUnique = new Map<string, Set<string>>();
|
||
for (const { row: validated, pkValue } of validatedRows) {
|
||
for (const colName of uniqueCols) {
|
||
const val = validated[colName];
|
||
if (val === undefined || val === null) continue;
|
||
const v = String(val);
|
||
let seen = batchUnique.get(colName);
|
||
if (!seen) {
|
||
seen = new Set<string>();
|
||
batchUnique.set(colName, seen);
|
||
}
|
||
if (seen.has(v)) {
|
||
throw new DatabaseError(
|
||
`Unique constraint violation on column "${colName}" in table "${tableName}"`,
|
||
'UNIQUE_VIOLATION',
|
||
);
|
||
}
|
||
seen.add(v);
|
||
await this.checkUnique(tableName, [colName], validated, pkValue);
|
||
}
|
||
}
|
||
|
||
for (const { row: validated, pkValue, key } of validatedRows) {
|
||
// PK 重复已在批预检阶段检查（v0.7.3），此处不再重复查询
|
||
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
// Within transaction: buffer to snapshot + MVCC version chain
|
||
this.txnSnapshot.set(key, validated);
|
||
this.mvcc.writeVersion(tableName, pkValue, validated, this.currentTxnId);
|
||
} else {
|
||
// Direct write to LSM (PK index)
|
||
this.lsm.put(key, validated);
|
||
}
|
||
|
||
// 更新二级索引
|
||
this.updateSecondaryIndexes(tableName, pkValue, validated, null);
|
||
|
||
pks.push(pkValue);
|
||
|
||
walRecords.push({
|
||
type: WALRecordType.INSERT,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName,
|
||
key: pkValue,
|
||
data: validated,
|
||
});
|
||
}
|
||
|
||
await this.wal.appendBatch(walRecords);
|
||
this.txnWalRecordCount += walRecords.length;
|
||
|
||
this.opCounter += rows.length;
|
||
this.checkMemoryBudget();
|
||
await this.checkpointManager.tick();
|
||
this.tryGC();
|
||
return pks;
|
||
}
|
||
|
||
async find(tableName: string, query: QueryPlan): Promise<Record<string, unknown>[]> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
|
||
let rows: Record<string, unknown>[];
|
||
|
||
// Try index lookup
|
||
const fastPath = await this.tryIndexLookup(tableName, query);
|
||
if (fastPath !== null) {
|
||
rows = fastPath;
|
||
} else {
|
||
rows = await this.getAllRows(tableName);
|
||
}
|
||
|
||
// v0.3.3: 事务内合并未提交快照（统一在 mergeTxnSnapshot 处理）
|
||
rows = this.mergeTxnSnapshot(tableName, rows);
|
||
|
||
// WHERE filter
|
||
if (query.where && Object.keys(query.where).length > 0) {
|
||
rows = rows.filter((row) => matchWhere(row, query.where!));
|
||
}
|
||
|
||
// ORDER
|
||
if (query.orderBy && query.orderBy.length > 0) {
|
||
rows = applyOrderBy(rows, query.orderBy);
|
||
}
|
||
|
||
// LIMIT/OFFSET
|
||
const offset = query.offset ?? 0;
|
||
const limit = query.limit ?? rows.length;
|
||
rows = rows.slice(offset, offset + limit);
|
||
|
||
// Column projection
|
||
if (query.columns && query.columns.length > 0 && query.columns[0] !== '*') {
|
||
rows = rows.map((row) => projectColumns(row, query.columns!));
|
||
}
|
||
|
||
// 查询完成，回收查询期间的临时缓存超限
|
||
this.trimAllCaches();
|
||
|
||
// v0.8.0: 行所有权 —— 返回副本，调用方不得改写存储（见 engine/interface.ts 约定）
|
||
return rows.map((row) => cloneRow(row));
|
||
}
|
||
|
||
async update(
|
||
tableName: string,
|
||
query: QueryPlan,
|
||
updates: Record<string, unknown>,
|
||
): Promise<number> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
// v0.7.4: 防御 —— QueryBuilder 直通引擎不经 Executor 子查询解析，
|
||
// 未解析的 $subquery/$col/$exists 在 matchWhere 中恒 false → 静默 0 行
|
||
if (containsUnresolvedSubqueries(query.where)) {
|
||
throw new DatabaseError(
|
||
'Unresolved subqueries/column references in UPDATE WHERE (use db.query() to execute subqueries)',
|
||
'NOT_SUPPORTED',
|
||
);
|
||
}
|
||
const schema = this.schemas.get(tableName)!;
|
||
const rows = await this.getAllRows(tableName);
|
||
let count = 0;
|
||
// v0.3.1: 批量 WAL 写入（组提交）
|
||
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
|
||
// v0.4.2-fix: ON UPDATE 级联环路保护
|
||
const visited = new Set<string>();
|
||
// v0.7.2: undefined 值视为"不更新该列"（保留旧值），null 显式置空
|
||
const cleanUpdates = stripUndefinedUpdates(updates);
|
||
|
||
// v0.7.4: 未知列显式报错 —— 此前 SET nonexistent = ... 被静默写入存储行
|
||
// （validateRow 只遍历 schema 列，脏列残留在行内并随 SSTable 持久化）
|
||
for (const col of Object.keys(cleanUpdates)) {
|
||
if (!schema.columns[col]) {
|
||
throw new DatabaseError(`Column "${col}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
|
||
}
|
||
}
|
||
|
||
// v0.6.2: 唯一约束 — 批量预加载本批更新涉及的唯一列索引范围（一次 drainChain）
|
||
const uniqueCols = this.uniqueColumns(tableName, schema);
|
||
for (const colName of uniqueCols) {
|
||
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`)!;
|
||
const ranges: [string, string][] = [];
|
||
if (cleanUpdates[colName] !== undefined && cleanUpdates[colName] !== null) {
|
||
const p = `${String(cleanUpdates[colName])}:`;
|
||
ranges.push([p, `${p}\uffff`]);
|
||
} else if (!(colName in cleanUpdates)) {
|
||
for (const row of rows) {
|
||
const val = row[colName];
|
||
if (val === undefined || val === null) continue;
|
||
const p = `${String(val)}:`;
|
||
ranges.push([p, `${p}\uffff`]);
|
||
}
|
||
}
|
||
await idxLsm.prefetchPrefixRanges(ranges);
|
||
}
|
||
|
||
// v0.7.2: 语句级原子性 — 两阶段（先全量预检，后执行）。
|
||
// 此前逐行"校验+写入"：第 N 行唯一冲突/校验失败抛错时，前 N-1 行已写入
|
||
// 且其 WAL 记录随 appendBatch 一起丢失 → 内存已改、WAL 无记录、调用方已收到错误
|
||
// （无事务下语句级部分提交 + 崩溃后进一步不一致）。
|
||
const planned: { row: Record<string, unknown>; pk: string; key: string; updated: Record<string, unknown>; newPk: string; pkChanged: boolean }[] = [];
|
||
const batchUnique: Map<string, Set<unknown>> = new Map();
|
||
|
||
// 阶段 1：全量预检（任何一行失败 → 整条语句不执行）
|
||
for (const row of rows) {
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const key = `${tableName}:${row[pkCol]}`;
|
||
if (query.where && Object.keys(query.where).length > 0 && !matchWhere(row, query.where)) continue;
|
||
|
||
const updated = { ...row, ...cleanUpdates };
|
||
this.validateRow(schema, updated);
|
||
|
||
// 批内唯一互查（索引尚未更新，两行同时改到同一新值需要互查兜底）
|
||
this.checkBatchUnique(tableName, uniqueCols, updated, batchUnique);
|
||
// v0.6.2: 唯一约束检查（排除自身旧索引条目：主键变更时旧条目仍以旧键存在）
|
||
await this.checkUnique(tableName, uniqueCols, updated, String(row[pkCol]));
|
||
|
||
// v0.4.2-fix: 支持更新主键 — 删除旧键 + 落新键 + WAL 两条记录
|
||
const newPk = String(updated[pkCol]);
|
||
const pkChanged = newPk !== String(row[pkCol]);
|
||
|
||
// v0.6.2-fix(P0): 主键变更撞已有主键 → 抛 DUPLICATE_KEY
|
||
// （此前静默覆盖另一行丢数据；与 MemoryEngine 对齐）
|
||
if (pkChanged) {
|
||
const newKey = `${tableName}:${newPk}`;
|
||
const existing = this.currentTxnId
|
||
? (this.txnSnapshot?.get(newKey) ?? await this.lsm.get(newKey))
|
||
: await this.lsm.get(newKey);
|
||
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
|
||
throw new DatabaseError(
|
||
`Duplicate primary key "${newPk}" in table "${tableName}" (cannot update key to existing value)`,
|
||
'DUPLICATE_KEY',
|
||
);
|
||
}
|
||
}
|
||
|
||
planned.push({ row, pk: String(row[pkCol]), key, updated, newPk, pkChanged });
|
||
}
|
||
// 阶段 1b：主键变更 RESTRICT / SET NULL+required 预检（任何修改前）
|
||
for (const p of planned) {
|
||
if (p.pkChanged) {
|
||
await this.checkForeignKeyUpdateRestrict(tableName, p.pk, p.newPk);
|
||
}
|
||
}
|
||
|
||
// 阶段 2：执行（预检已通过，此阶段不再抛校验类错误）
|
||
for (const { row, pk, key, updated, newPk, pkChanged } of planned) {
|
||
if (pkChanged) {
|
||
// ON UPDATE 外键级联（RESTRICT 抛错 / CASCADE / SET NULL）
|
||
await this.applyForeignKeyUpdateRules(tableName, pk, newPk, walRecords, visited);
|
||
}
|
||
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
if (pkChanged) {
|
||
this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record<string, unknown>);
|
||
this.mvcc.deleteVersion(tableName, pk, this.currentTxnId);
|
||
}
|
||
this.txnSnapshot.set(`${tableName}:${newPk}`, updated);
|
||
this.mvcc.writeVersion(tableName, newPk, updated, this.currentTxnId);
|
||
} else {
|
||
if (pkChanged) this.lsm.delete(key);
|
||
this.lsm.put(`${tableName}:${newPk}`, updated);
|
||
}
|
||
count++;
|
||
|
||
if (pkChanged) {
|
||
walRecords.push({
|
||
type: WALRecordType.DELETE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName,
|
||
key: pk,
|
||
});
|
||
}
|
||
walRecords.push({
|
||
type: WALRecordType.UPDATE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName,
|
||
key: newPk,
|
||
data: updated,
|
||
});
|
||
|
||
// 更新二级索引（主键变更时旧索引条目一并清理）
|
||
// v0.6.2-fix: 此前非主键更新不传旧行 → 旧索引条目残留
|
||
// （唯一性检查误报 / 索引存储膨胀）；现在统一传旧行清理旧值
|
||
this.updateSecondaryIndexes(tableName, newPk, updated, row);
|
||
}
|
||
|
||
await this.wal.appendBatch(walRecords);
|
||
this.txnWalRecordCount += walRecords.length;
|
||
|
||
this.opCounter += count;
|
||
await this.checkpointManager.tick();
|
||
this.trimAllCaches();
|
||
return count;
|
||
}
|
||
|
||
/**
|
||
* v0.7.2: 批内唯一互查 — 两条行在同一语句中更新到同一唯一值时的兜底检查
|
||
* （阶段 1 中索引尚未反映本语句的变更）。
|
||
*/
|
||
private checkBatchUnique(
|
||
tableName: string,
|
||
uniqueCols: string[],
|
||
updated: Record<string, unknown>,
|
||
batchUnique: Map<string, Set<unknown>>,
|
||
): void {
|
||
for (const colName of uniqueCols) {
|
||
const value = updated[colName];
|
||
if (value === undefined || value === null) continue;
|
||
let seen = batchUnique.get(colName);
|
||
if (!seen) {
|
||
seen = new Set<unknown>();
|
||
batchUnique.set(colName, seen);
|
||
}
|
||
if (seen.has(value)) {
|
||
throw new DatabaseError(
|
||
`Unique constraint violation on column "${colName}" in table "${tableName}"`,
|
||
'UNIQUE_VIOLATION',
|
||
);
|
||
}
|
||
seen.add(value);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.7.2: ON UPDATE 外键预检 — 从 applyForeignKeyUpdateRules 提取（两阶段 update 用）：
|
||
* RESTRICT 存在依赖行抛错；SET NULL 撞 required 列同样整体拒绝。
|
||
*/
|
||
private async checkForeignKeyUpdateRestrict(tableName: string, oldPk: string, _newPk: string): Promise<void> {
|
||
for (const [refTableName, refSchema] of this.schemas) {
|
||
if (refTableName === tableName) continue;
|
||
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
|
||
if (!colDef.references || !colDef.onUpdate) continue;
|
||
const [refTable] = colDef.references.split('.');
|
||
if (refTable !== tableName) continue;
|
||
if (colDef.onUpdate === 'RESTRICT' || (colDef.onUpdate === 'SET NULL' && colDef.required)) {
|
||
const refRows = await this.getAllRows(refTableName);
|
||
for (const refRow of refRows) {
|
||
if (String(refRow[colName]) === oldPk) {
|
||
const reason = colDef.onUpdate === 'RESTRICT'
|
||
? `foreign key "${colName}" in "${refTableName}" has dependent rows`
|
||
: `foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`;
|
||
throw new DatabaseError(
|
||
`Cannot update "${tableName}" key "${oldPk}": ${reason}`,
|
||
'FOREIGN_KEY_VIOLATION',
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.4.2-fix: ON UPDATE 外键级联 — 主键 oldPk → newPk 时处理引用表。
|
||
* RESTRICT 抛错 / CASCADE 更新 FK / SET NULL 置空（含索引与 WAL 记录）。
|
||
* 两阶段：先全量 RESTRICT 检查，再执行级联。
|
||
*/
|
||
private async applyForeignKeyUpdateRules(
|
||
tableName: string,
|
||
oldPk: string,
|
||
newPk: string,
|
||
walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[],
|
||
visited: Set<string>,
|
||
): Promise<void> {
|
||
const visitKey = `${tableName}:${oldPk}`;
|
||
if (visited.has(visitKey)) return;
|
||
visited.add(visitKey);
|
||
|
||
// v0.7.3-perf: 删除冗余的阶段 1 RESTRICT 扫描 —— checkForeignKeyUpdateRestrict
|
||
// 已在两阶段 update 预检（阶段 1b）覆盖 RESTRICT 与 SET NULL+required，
|
||
// 此处任何修改前重复全表扫描纯属浪费。直接执行 CASCADE / SET NULL。
|
||
for (const [refTableName, refSchema] of this.schemas) {
|
||
if (refTableName === tableName) continue;
|
||
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
|
||
if (!colDef.references || !colDef.onUpdate) continue;
|
||
const [refTable] = colDef.references.split('.');
|
||
if (refTable !== tableName) continue;
|
||
if (colDef.onUpdate !== 'CASCADE' && colDef.onUpdate !== 'SET NULL') continue;
|
||
const refRows = await this.getAllRows(refTableName);
|
||
for (const refRow of refRows) {
|
||
if (String(refRow[colName]) !== oldPk) continue;
|
||
const refPkCol = this.tablePKs.get(refTableName)!;
|
||
const refPk = String(refRow[refPkCol]);
|
||
const updatedRef = { ...refRow, [colName]: colDef.onUpdate === 'CASCADE' ? newPk : null };
|
||
const refKey = `${refTableName}:${refPk}`;
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
this.txnSnapshot.set(refKey, updatedRef);
|
||
this.mvcc.writeVersion(refTableName, refPk, updatedRef, this.currentTxnId);
|
||
} else {
|
||
this.lsm.put(refKey, updatedRef);
|
||
}
|
||
this.updateSecondaryIndexes(refTableName, refPk, updatedRef, refRow);
|
||
walRecords.push({
|
||
type: WALRecordType.UPDATE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName: refTableName,
|
||
key: refPk,
|
||
data: updatedRef,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
async delete(tableName: string, query: QueryPlan): Promise<number> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
// v0.7.4: 防御 —— QueryBuilder 直通引擎不经 Executor 子查询解析，
|
||
// 未解析的 $subquery/$col/$exists 在 matchWhere 中恒 false → 静默 0 行
|
||
if (containsUnresolvedSubqueries(query.where)) {
|
||
throw new DatabaseError(
|
||
'Unresolved subqueries/column references in DELETE WHERE (use db.query() to execute subqueries)',
|
||
'NOT_SUPPORTED',
|
||
);
|
||
}
|
||
|
||
const rows = await this.getAllRows(tableName);
|
||
let count = 0;
|
||
// v0.3.1: 批量 WAL 写入（组提交）
|
||
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
|
||
// v0.4.1: 外键级联（环路保护）
|
||
const visited = new Set<string>();
|
||
|
||
// v0.6.3-fix: 级联两阶段 —— 先对全部匹配行做 RESTRICT 预检（沿 CASCADE 链递归），
|
||
// 否则第 N 行 RESTRICT 抛错时前 N-1 行的级联已执行 → 无事务部分级联（数据不一致）
|
||
const matchedPks: string[] = [];
|
||
for (const row of rows) {
|
||
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
|
||
matchedPks.push(String(row[this.tablePKs.get(tableName)!]));
|
||
}
|
||
}
|
||
const restrictVisited = new Set<string>();
|
||
for (const pkValue of matchedPks) {
|
||
await this.checkCascadeRestrict(tableName, pkValue, restrictVisited);
|
||
}
|
||
|
||
for (const row of rows) {
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const key = `${tableName}:${row[pkCol]}`;
|
||
|
||
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
|
||
// v0.4.1: 外键规则（RESTRICT 抛错 / CASCADE 递归删 / SET NULL 置空）
|
||
count += await this.applyForeignKeyRules(tableName, String(row[pkCol]), walRecords, visited);
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
// Buffer delete in snapshot + MVCC tombstone
|
||
this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record<string, unknown>);
|
||
this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId);
|
||
} else {
|
||
this.lsm.delete(key);
|
||
}
|
||
count++;
|
||
|
||
walRecords.push({
|
||
type: WALRecordType.DELETE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName,
|
||
key: String(row[pkCol]),
|
||
});
|
||
|
||
// 移除二级索引
|
||
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
|
||
}
|
||
}
|
||
|
||
await this.wal.appendBatch(walRecords);
|
||
this.txnWalRecordCount += walRecords.length;
|
||
|
||
this.opCounter += count;
|
||
await this.checkpointManager.tick();
|
||
this.trimAllCaches();
|
||
return count;
|
||
}
|
||
|
||
/**
|
||
* v0.6.3: RESTRICT 预检（delete 级联两阶段之一，与 MemoryEngine 对齐）。
|
||
* 递归沿 CASCADE 链检查引用表：RESTRICT 引用存在依赖行则抛 FOREIGN_KEY_VIOLATION。
|
||
*/
|
||
private async checkCascadeRestrict(tableName: string, pkValue: string, visited: Set<string>): Promise<void> {
|
||
const visitKey = `${tableName}:${pkValue}`;
|
||
if (visited.has(visitKey)) return;
|
||
visited.add(visitKey);
|
||
|
||
for (const [refTableName, refSchema] of this.schemas) {
|
||
if (refTableName === tableName) continue;
|
||
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
|
||
if (!colDef.references || !colDef.onDelete) continue;
|
||
const [refTable] = colDef.references.split('.');
|
||
if (refTable !== tableName) continue;
|
||
const refRows = await this.getAllRows(refTableName);
|
||
const matched = refRows.filter((r) => String(r[colName]) === pkValue);
|
||
if (colDef.onDelete === 'RESTRICT' && matched.length > 0) {
|
||
throw new DatabaseError(
|
||
`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`,
|
||
'FOREIGN_KEY_VIOLATION',
|
||
);
|
||
}
|
||
// v0.7.2: SET NULL 到 required 列违反约束 —— 预检阶段整体拒绝
|
||
if (colDef.onDelete === 'SET NULL' && colDef.required && matched.length > 0) {
|
||
throw new DatabaseError(
|
||
`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`,
|
||
'FOREIGN_KEY_VIOLATION',
|
||
);
|
||
}
|
||
if (colDef.onDelete === 'CASCADE') {
|
||
const refPkCol = this.tablePKs.get(refTableName)!;
|
||
for (const refRow of matched) {
|
||
await this.checkCascadeRestrict(refTableName, String(refRow[refPkCol]), visited);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.4.1: 外键级联规则 — 对齐 MemoryEngine.cascadeDelete 行为。
|
||
* 删除 tableName 主键为 pkValue 的行前，检查引用它的所有表：
|
||
* - RESTRICT: 存在引用行 → 抛 FOREIGN_KEY_VIOLATION
|
||
* - CASCADE: 递归删除引用行（含索引/WAL）
|
||
* - SET NULL: 引用行外键列置 null（含索引/WAL）
|
||
* @returns 级联影响的行数（CASCADE 删除行数 + SET NULL 更新行数）
|
||
*/
|
||
private async applyForeignKeyRules(
|
||
tableName: string,
|
||
pkValue: string,
|
||
walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[],
|
||
visited: Set<string>,
|
||
): Promise<number> {
|
||
let total = 0;
|
||
const visitKey = `${tableName}:${pkValue}`;
|
||
if (visited.has(visitKey)) return 0;
|
||
visited.add(visitKey);
|
||
|
||
for (const [refTableName, refSchema] of this.schemas) {
|
||
if (refTableName === tableName) continue;
|
||
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
|
||
if (!colDef.references || !colDef.onDelete) continue;
|
||
const [refTable] = colDef.references.split('.');
|
||
if (refTable !== tableName) continue;
|
||
|
||
const refRows = await this.getAllRows(refTableName);
|
||
const matched = refRows.filter((r) => String(r[colName]) === pkValue);
|
||
|
||
if (colDef.onDelete === 'RESTRICT' && matched.length > 0) {
|
||
throw new DatabaseError(
|
||
`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`,
|
||
'FOREIGN_KEY_VIOLATION',
|
||
);
|
||
}
|
||
|
||
if (colDef.onDelete === 'CASCADE') {
|
||
const refPkCol = this.tablePKs.get(refTableName)!;
|
||
for (const refRow of matched) {
|
||
const refPk = String(refRow[refPkCol]);
|
||
// 递归级联（先处理更深层引用）
|
||
total += await this.applyForeignKeyRules(refTableName, refPk, walRecords, visited);
|
||
// 删除引用行
|
||
const refKey = `${refTableName}:${refPk}`;
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
this.txnSnapshot.set(refKey, { __txn_deleted: true } as unknown as Record<string, unknown>);
|
||
this.mvcc.deleteVersion(refTableName, refPk, this.currentTxnId);
|
||
} else {
|
||
this.lsm.delete(refKey);
|
||
}
|
||
this.updateSecondaryIndexes(refTableName, refPk, null, refRow);
|
||
walRecords.push({
|
||
type: WALRecordType.DELETE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName: refTableName,
|
||
key: refPk,
|
||
});
|
||
total++;
|
||
}
|
||
} else if (colDef.onDelete === 'SET NULL') {
|
||
const refPkCol = this.tablePKs.get(refTableName)!;
|
||
for (const refRow of matched) {
|
||
const refPk = String(refRow[refPkCol]);
|
||
const updated = { ...refRow, [colName]: null };
|
||
const refKey = `${refTableName}:${refPk}`;
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
this.txnSnapshot.set(refKey, updated);
|
||
this.mvcc.writeVersion(refTableName, refPk, updated, this.currentTxnId);
|
||
} else {
|
||
this.lsm.put(refKey, updated);
|
||
}
|
||
this.updateSecondaryIndexes(refTableName, refPk, updated, refRow);
|
||
walRecords.push({
|
||
type: WALRecordType.UPDATE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName: refTableName,
|
||
key: refPk,
|
||
data: updated,
|
||
});
|
||
// 对齐 Memory 语义：SET NULL 不影响返回的删除行数
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return total;
|
||
}
|
||
|
||
/**
|
||
* v0.4.0: 流式查询 — 逐行回调，不物化结果数组。
|
||
* 全表路径走 LSM rangeScanLazy 惰性扫描；索引等值/范围路径复用 tryIndexLookup。
|
||
* 事务中回退物化（快照合并需要全量行集）。
|
||
*/
|
||
async findStream(
|
||
tableName: string,
|
||
query: QueryPlan,
|
||
onRow: (row: Record<string, unknown>) => void,
|
||
): Promise<number> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
|
||
const hasWhere = !!(query.where && Object.keys(query.where).length > 0);
|
||
const project = query.columns && query.columns.length > 0 && query.columns[0] !== '*'
|
||
? (row: Record<string, unknown>) => projectColumns(row, query.columns!)
|
||
: null;
|
||
const limit = query.limit ?? Infinity;
|
||
const offset = query.offset ?? 0;
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const prefix = `${tableName}:`;
|
||
let count = 0;
|
||
let skipped = 0;
|
||
|
||
const emit = (row: Record<string, unknown>): boolean => {
|
||
if (hasWhere && !matchWhere(row, query.where!)) return true;
|
||
if (skipped < offset) { skipped++; return true; }
|
||
onRow(project ? project(row) : cloneRow(row));
|
||
count++;
|
||
return count < limit;
|
||
};
|
||
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
// 事务中：物化后逐行回调（快照合并需要全量行集）
|
||
const rows = await this.find(tableName, { ...query, orderBy: undefined, limit: undefined, offset: undefined });
|
||
for (const row of rows) {
|
||
onRow(project ? project(row) : cloneRow(row));
|
||
}
|
||
return rows.length;
|
||
}
|
||
|
||
// 索引路径：等值/范围查找（结果行已过滤，直接回调）
|
||
const fastPath = await this.tryIndexLookup(tableName, query);
|
||
if (fastPath !== null) {
|
||
for (const row of fastPath) {
|
||
if (!emit(row)) break;
|
||
}
|
||
return count;
|
||
}
|
||
|
||
// 全表惰性扫描（含 WHERE 过滤，不物化；v0.7.4: callback 返回 false 提前终止，
|
||
// 未消费的 SSTable 块 / 子树不再解析 —— 真流式，大表 limit 内存 O(1)）
|
||
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
|
||
await this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => {
|
||
if (count >= limit) return false;
|
||
const row = { ...value };
|
||
row[pkCol] = key.slice(prefix.length);
|
||
return emit(row);
|
||
});
|
||
return count;
|
||
}
|
||
|
||
async count(tableName: string, query?: QueryPlan): Promise<number> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
const rows = await this.getAllRows(tableName);
|
||
this.trimAllCaches();
|
||
if (!query?.where || Object.keys(query.where).length === 0) return rows.length;
|
||
return rows.filter((row) => matchWhere(row, query.where!)).length;
|
||
}
|
||
|
||
async clear(tableName: string): Promise<void> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
const rows = await this.getAllRows(tableName);
|
||
// v0.3.3: 事务内清空走快照（删除标记），提交时生效；并写入 WAL
|
||
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
|
||
for (const row of rows) {
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const key = `${tableName}:${row[pkCol]}`;
|
||
if (this.currentTxnId && this.txnSnapshot) {
|
||
this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record<string, unknown>);
|
||
this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId);
|
||
} else {
|
||
this.lsm.delete(key);
|
||
}
|
||
walRecords.push({
|
||
type: WALRecordType.DELETE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName,
|
||
key: String(row[pkCol]),
|
||
});
|
||
// 移除二级索引
|
||
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
|
||
}
|
||
await this.wal.appendBatch(walRecords);
|
||
this.txnWalRecordCount += walRecords.length;
|
||
this.opCounter += rows.length;
|
||
await this.checkpointManager.tick();
|
||
this.tryGC();
|
||
}
|
||
|
||
// ---- ALTER TABLE（v0.4.1） ----
|
||
|
||
/**
|
||
* v0.4.1: ALTER TABLE — 结构变更真正生效于存储：
|
||
* - ADD: 持久化 schema（persistSchemas），行无需修改
|
||
* - DROP: 持久化 schema + 遍历主 LSM 重写所有行（移除该列键）+ WAL UPDATE 记录
|
||
* （通用路径 getTableSchema 返回副本，Executor 的引用修改对 Aria 无效）
|
||
*/
|
||
async alterTable(
|
||
tableName: string,
|
||
action: 'ADD' | 'DROP',
|
||
column: import('../../constants').ColumnDef & { name: string },
|
||
): Promise<void> {
|
||
this.ensureOpen();
|
||
this.ensureNoDDLInTransaction('ALTER TABLE');
|
||
this.ensureTable(tableName);
|
||
const schema = this.schemas.get(tableName)!;
|
||
|
||
if (action === 'ADD') {
|
||
if (schema.columns[column.name]) {
|
||
throw new DatabaseError(`Column "${column.name}" already exists in table "${tableName}"`, 'COLUMN_EXISTS');
|
||
}
|
||
schema.columns[column.name] = column;
|
||
await this.persistSchemas();
|
||
return;
|
||
}
|
||
|
||
// DROP
|
||
if (!schema.columns[column.name]) {
|
||
throw new DatabaseError(`Column "${column.name}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
|
||
}
|
||
// v0.4.2-fix: 被删列是索引列 → 先清理索引 LSM（残留会导致后续同名列索引脏数据）
|
||
if (schema.columns[column.name].index || schema.columns[column.name].unique) {
|
||
const idxKey = `${tableName}:idx:${column.name}`;
|
||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||
if (idxLsm) {
|
||
try {
|
||
await idxLsm.clear();
|
||
} catch { /* 清理失败不阻塞 */ }
|
||
this.secondaryIndexes.delete(idxKey);
|
||
}
|
||
}
|
||
delete schema.columns[column.name];
|
||
await this.persistSchemas();
|
||
|
||
// 重写主 LSM：移除所有行的该列键（find 副本无法就地删除，必须重写存储）
|
||
const prefix = `${tableName}:`;
|
||
const endKey = `${prefix}\uffff`;
|
||
await this.lsm.prefetchRange(prefix, endKey);
|
||
const entries = await this.lsm.rangeScan(prefix, endKey);
|
||
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
|
||
for (const [key, value] of entries) {
|
||
if (!(column.name in value)) continue;
|
||
const updated = { ...value };
|
||
delete updated[column.name];
|
||
this.lsm.put(key, updated);
|
||
// 二级索引列被删时同步清理索引
|
||
const pk = key.slice(prefix.length);
|
||
this.updateSecondaryIndexes(tableName, pk, updated, value);
|
||
walRecords.push({
|
||
type: WALRecordType.UPDATE,
|
||
txnId: this.currentTxnId ?? 0,
|
||
tableName,
|
||
key: pk,
|
||
data: updated,
|
||
});
|
||
}
|
||
await this.wal.appendBatch(walRecords);
|
||
this.txnWalRecordCount += walRecords.length;
|
||
this.opCounter += walRecords.length;
|
||
await this.checkpointManager.tick();
|
||
this.trimAllCaches();
|
||
}
|
||
|
||
async createIndex(tableName: string, column: string, unique?: boolean): Promise<void> {
|
||
this.ensureOpen();
|
||
this.ensureNoDDLInTransaction('CREATE INDEX');
|
||
this.ensureTable(tableName);
|
||
const schema = this.schemas.get(tableName)!;
|
||
const colDef = schema.columns[column];
|
||
if (!colDef) throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
|
||
const idxKey = `${tableName}:idx:${column}`;
|
||
// v0.4.2-fix: 以索引 LSM 是否已建为准（schema 标记可能因重启恢复而存在，
|
||
// 但索引 LSM 未恢复 → 此前静默 return 导致索引永久缺失）
|
||
if (this.secondaryIndexes.has(idxKey)) return;
|
||
|
||
const idxLsm = new LSM({
|
||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||
blockSize: this.config.pageSize,
|
||
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
|
||
sstableStore: this.createSSTableStore(`idx_${tableName}_${column}`),
|
||
});
|
||
await idxLsm.init();
|
||
this.secondaryIndexes.set(idxKey, idxLsm);
|
||
|
||
try {
|
||
// 从主 LSM 重建索引数据
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const rows = await this.getAllRows(tableName);
|
||
const seen = new Set<string>();
|
||
for (const row of rows) {
|
||
const value = row[column];
|
||
if (value !== undefined && value !== null) {
|
||
const v = String(value);
|
||
// v0.7.3: UNIQUE 索引回填校验存量唯一性 —— 此前重复数据静默建索引
|
||
// （SQLite 语义应报错），与 MemoryEngine 对齐
|
||
if (unique && seen.has(v)) {
|
||
throw new DatabaseError(
|
||
`Unique index on column "${column}" in table "${tableName}" cannot be created: duplicate value "${v}"`,
|
||
'UNIQUE_VIOLATION',
|
||
);
|
||
}
|
||
seen.add(v);
|
||
idxLsm.put(`${v}:${row[pkCol]}`, { pk: row[pkCol] });
|
||
}
|
||
}
|
||
await idxLsm.flush();
|
||
} catch (error) {
|
||
// 回填失败（唯一冲突）：清理半初始化索引（内存 + 存储），标志未落，保持原子语义
|
||
this.secondaryIndexes.delete(idxKey);
|
||
try { await idxLsm.clear(); } catch { /* 清理失败不阻塞 */ }
|
||
throw error;
|
||
}
|
||
colDef.index = true;
|
||
if (unique) {
|
||
colDef.unique = true;
|
||
// v0.7.4: 记录唯一约束来源（DROP INDEX 时可解除；建表约束不可）
|
||
this.uniqueIndexCols.add(`${tableName}:${column}`);
|
||
}
|
||
await this.persistSchemas();
|
||
}
|
||
|
||
async dropIndex(tableName: string, column: string, _indexName?: string): Promise<void> {
|
||
this.ensureOpen();
|
||
this.ensureNoDDLInTransaction('DROP INDEX');
|
||
this.ensureTable(tableName);
|
||
const schema = this.schemas.get(tableName)!;
|
||
const colDef = schema.columns[column];
|
||
if (!colDef) throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
|
||
// 主键索引不可删除（PK 查找依赖主 LSM）
|
||
if (colDef.primaryKey) {
|
||
throw new DatabaseError(`Cannot drop primary key index on column "${column}"`, 'NOT_SUPPORTED');
|
||
}
|
||
// v0.4.1: DROP 不存在的索引应报错（此前静默成功）
|
||
if (!colDef.index && !colDef.unique && !this.secondaryIndexes.has(`${tableName}:idx:${column}`)) {
|
||
throw new DatabaseError(`Index on column "${column}" does not exist in table "${tableName}"`, 'INDEX_NOT_FOUND');
|
||
}
|
||
// v0.7.4: 建表 UNIQUE 约束不可通过 DROP INDEX 解除 —— 此前 colDef.unique = false
|
||
// 静默解除约束（重启后 persistSchemas 使约束永久消失）。对齐 SQLite 语义：
|
||
// 约束随建表存在，解除需重建表；仅 CREATE UNIQUE INDEX 添加的约束可随索引删除。
|
||
const uniqueKey = `${tableName}:${column}`;
|
||
if (colDef.unique && !this.uniqueIndexCols.has(uniqueKey)) {
|
||
throw new DatabaseError(
|
||
`Cannot drop index on column "${column}" in table "${tableName}": ` +
|
||
'UNIQUE constraint defined at table creation must be removed by recreating the table',
|
||
'NOT_SUPPORTED',
|
||
);
|
||
}
|
||
colDef.index = false;
|
||
colDef.unique = false;
|
||
this.uniqueIndexCols.delete(uniqueKey);
|
||
|
||
const idxKey = `${tableName}:idx:${column}`;
|
||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||
if (idxLsm) {
|
||
await idxLsm.clear();
|
||
this.secondaryIndexes.delete(idxKey);
|
||
}
|
||
await this.persistSchemas();
|
||
}
|
||
|
||
// =======================================================================
|
||
// 事务
|
||
// =======================================================================
|
||
|
||
async beginTransaction(): Promise<void> {
|
||
if (this.currentTxnId) throw new DatabaseError('Transaction already in progress', 'TX_ACTIVE');
|
||
this.currentTxnId = this.mvcc.beginTransaction();
|
||
this.txnSnapshot = new Map();
|
||
// v0.8.0: 事务内 WAL 记录计数从 0 开始（保存点边界依赖它）
|
||
this.txnWalRecordCount = 0;
|
||
|
||
// v0.7.3-fix: WAL BEGIN 写失败回滚内存事务状态 —— 此前 append 抛错（full 模式）
|
||
// 时 currentTxnId 已设置 → TX_ACTIVE 永久泄漏（后续无法开始新事务）。
|
||
// 回滚 mvcc 登记 + 快照后重抛，调用方可重试。
|
||
try {
|
||
await this.wal.append({
|
||
type: WALRecordType.BEGIN,
|
||
txnId: this.currentTxnId,
|
||
tableName: '',
|
||
key: '',
|
||
});
|
||
this.txnWalRecordCount = 1; // BEGIN 本身占一条
|
||
} catch (error) {
|
||
this.mvcc.rollbackTransaction(this.currentTxnId);
|
||
this.currentTxnId = null;
|
||
this.txnSnapshot = null;
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async commitTransaction(): Promise<void> {
|
||
if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE');
|
||
|
||
// v0.4.3-fix: 先持久化 WAL COMMIT，再合并快照到 LSM —
|
||
// 崩溃在 WAL 提交后、快照合并前：恢复时 WAL 重放数据，重启一致；
|
||
// 崩溃在 WAL 提交前：commitTransaction 尚未返回，事务视为未提交（可回滚）
|
||
await this.wal.append({
|
||
type: WALRecordType.COMMIT,
|
||
txnId: this.currentTxnId,
|
||
tableName: '',
|
||
key: '',
|
||
});
|
||
await this.wal.flush();
|
||
|
||
if (this.txnSnapshot) {
|
||
for (const [key, value] of this.txnSnapshot) {
|
||
if ((value as unknown as Record<string, unknown>).__txn_deleted) {
|
||
this.lsm.delete(key);
|
||
} else {
|
||
this.lsm.put(key, value);
|
||
}
|
||
}
|
||
}
|
||
|
||
this.mvcc.commitTransaction(this.currentTxnId);
|
||
|
||
// v0.8.0 根治：事务结束时清空 savepoint 表。
|
||
// 此前 commit/rollback **都不清理** savepoints，于是：
|
||
// 1. 上一个事务的名字仍被占用 → 新事务 SAVEPOINT 同名报 "already exists"；
|
||
// 2. 新事务 ROLLBACK TO 陈旧名字会拿到旧事务快照，把当前事务的写入静默替换。
|
||
this.savepoints.clear();
|
||
this.savepointWalBoundary.clear();
|
||
this.txnWalRecordCount = 0;
|
||
this.currentTxnId = null;
|
||
this.txnSnapshot = null;
|
||
}
|
||
|
||
async rollbackTransaction(): Promise<void> {
|
||
if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE');
|
||
|
||
const txnId = this.currentTxnId;
|
||
|
||
// v0.7.3-fix: 先持久化 WAL ROLLBACK，再回滚内存 —— 与 commitTransaction 的
|
||
// "WAL 领先内存"（v0.4.3-fix）对齐。此前内存先回滚、ROLLBACK 记录后写：
|
||
// full 模式写失败时崩溃重放无 ROLLBACK 记录 → 已回滚事务的数据复活。
|
||
// 现在写失败 → 内存未回滚、事务仍活跃（调用方可重试），崩溃后重放
|
||
// 看到 ROLLBACK 记录同样不会复活数据。
|
||
await this.wal.append({
|
||
type: WALRecordType.ROLLBACK,
|
||
txnId,
|
||
tableName: '',
|
||
key: '',
|
||
});
|
||
|
||
// v0.3.3: 记录事务涉及的表（用于回滚后重建索引，消除索引残留）
|
||
const affectedTables = new Set<string>();
|
||
if (this.txnSnapshot) {
|
||
for (const key of this.txnSnapshot.keys()) {
|
||
const idx = key.indexOf(':');
|
||
if (idx > 0) affectedTables.add(key.slice(0, idx));
|
||
}
|
||
}
|
||
|
||
this.mvcc.rollbackTransaction(txnId);
|
||
this.txnSnapshot = null;
|
||
|
||
// v0.8.0：事务结束清空 savepoint（理由同 commitTransaction）
|
||
this.savepoints.clear();
|
||
this.savepointWalBoundary.clear();
|
||
this.txnWalRecordCount = 0;
|
||
this.currentTxnId = null;
|
||
|
||
// v0.3.3: 事务内直接写入了二级索引 LSM，回滚后全量重建受影响表的索引
|
||
for (const tableName of affectedTables) {
|
||
if (this.schemas.has(tableName)) {
|
||
await this.reindexTable(tableName);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---- Savepoint 嵌套事务 ----
|
||
|
||
private savepoints: Map<string, { txnId: number; snapshot: Map<string, Record<string, unknown>> | null }> = new Map();
|
||
|
||
/**
|
||
* v0.8.0: 当前事务在 WAL 中已成功追加的记录条数。
|
||
*
|
||
* 用途：保存点需要记录"回滚后应保留到哪一条"，否则恢复时无法区分
|
||
* "保存点之前的写入"（应保留）与"保存点之后的写入"（应丢弃）。
|
||
*/
|
||
private txnWalRecordCount = 0;
|
||
|
||
/** 保存点 → 该保存点时事务的 WAL 记录边界 */
|
||
private savepointWalBoundary = new Map<string, number>();
|
||
|
||
async savepoint(name: string): Promise<void> {
|
||
if (!this.currentTxnId) throw new DatabaseError('No active transaction for savepoint', 'TX_NONE');
|
||
if (this.savepoints.has(name)) throw new DatabaseError(`Savepoint "${name}" already exists`, 'SAVEPOINT_EXISTS');
|
||
// 保存当前事务快照
|
||
this.savepoints.set(name, {
|
||
txnId: this.currentTxnId,
|
||
snapshot: this.txnSnapshot ? new Map(this.txnSnapshot) : null,
|
||
});
|
||
// v0.8.0: 记录边界（该事务已追加的记录条数）—— 见 txnWalRecordCount 说明
|
||
this.savepointWalBoundary.set(`${this.currentTxnId}:${name}`, this.txnWalRecordCount);
|
||
}
|
||
|
||
async rollbackToSavepoint(name: string): Promise<void> {
|
||
const sp = this.savepoints.get(name);
|
||
if (!sp) throw new DatabaseError(`Savepoint "${name}" not found`, 'SAVEPOINT_NOT_FOUND');
|
||
// v0.8.0 根治：savepoint 归属校验。
|
||
// 此前只按名字查找，而 savepoints 在 commit/rollback 时**从不清空** ——
|
||
// 上一个事务遗留的 savepoint 会让当前事务的写入被旧事务快照静默替换
|
||
// （实测：本事务 update 后 ROLLBACK TO 陈旧名 + COMMIT，写入凭空消失）。
|
||
if (sp.txnId !== this.currentTxnId) {
|
||
throw new DatabaseError(
|
||
`Savepoint "${name}" belongs to a different transaction (stale savepoint)`,
|
||
'SAVEPOINT_NOT_FOUND',
|
||
);
|
||
}
|
||
// v0.8.0: 先写 WAL 标记再改内存（与 commit/rollback 的"WAL 领先内存"一致）——
|
||
// 否则崩溃恢复会把该事务在 savepoint 之前写入的记录重新应用，已回滚的行复活。
|
||
const boundary = this.savepointWalBoundary.get(`${this.currentTxnId}:${name}`) ?? 0;
|
||
await this.wal.append({
|
||
type: WALRecordType.SAVEPOINT_ROLLBACK,
|
||
txnId: this.currentTxnId!,
|
||
tableName: '',
|
||
key: '',
|
||
data: { replayFromIndex: boundary },
|
||
});
|
||
this.txnWalRecordCount++;
|
||
this.savepointWalBoundary.set(`${this.currentTxnId}:${name}`, boundary);
|
||
// v0.6.3-fix: 记录当前快照涉及的表（回滚后重建索引）—— 此前事务内直写
|
||
// 索引 LSM，savepoint 回滚只还原快照 → savepoint 之后的索引条目残留
|
||
const affectedTables = new Set<string>();
|
||
if (this.txnSnapshot) {
|
||
for (const key of this.txnSnapshot.keys()) {
|
||
const idx = key.indexOf(':');
|
||
if (idx > 0) affectedTables.add(key.slice(0, idx));
|
||
}
|
||
}
|
||
// 恢复到 savepoint 时的快照
|
||
this.txnSnapshot = sp.snapshot ? new Map(sp.snapshot) : null;
|
||
// v0.3.3: 清理该事务在 MVCC 版本链中的全部记录（快照已含正确数据，
|
||
// 版本链仅作 undo 记录，清空后 commit 时 LSM 写入与快照保持一致）
|
||
this.mvcc.discardVersions(this.currentTxnId!);
|
||
// 清除此 savepoint 之后的所有 savepoint
|
||
let found = false;
|
||
for (const [k] of this.savepoints) {
|
||
if (k === name) { found = true; continue; }
|
||
if (found) this.savepoints.delete(k);
|
||
}
|
||
// v0.6.3-fix: 重建受影响表二级索引（消除 savepoint 之后的过期索引条目）
|
||
for (const tableName of affectedTables) {
|
||
if (this.schemas.has(tableName)) {
|
||
await this.reindexTable(tableName);
|
||
}
|
||
}
|
||
}
|
||
|
||
async releaseSavepoint(name: string): Promise<void> {
|
||
if (!this.savepoints.has(name)) throw new DatabaseError(`Savepoint "${name}" not found`, 'SAVEPOINT_NOT_FOUND');
|
||
this.savepoints.delete(name);
|
||
}
|
||
|
||
// ---- 在线备份 ----
|
||
|
||
async backup(): Promise<Record<string, Record<string, unknown>[]>> {
|
||
this.ensureOpen();
|
||
const result: Record<string, Record<string, unknown>[]> = {};
|
||
for (const tableName of this.schemas.keys()) {
|
||
result[tableName] = await this.getAllRows(tableName);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
// =======================================================================
|
||
// 内部
|
||
// =======================================================================
|
||
|
||
private async getAllRows(tableName: string): Promise<Record<string, unknown>[]> {
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const prefix = `${tableName}:`;
|
||
// 预加载范围内涉及的 SSTable，避免 rangeScan 时缓存未命中静默丢数据
|
||
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
|
||
const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||
const rows = entries.map(([key, value]) => {
|
||
// v0.8.0: 深拷贝（此前 `{ ...value }` 只做浅拷贝，嵌套 json 值仍与 LSM
|
||
// 内部对象共享引用 —— 调用方改 rows[0].nested.a 会改写存储）
|
||
const row = cloneRow(value);
|
||
row[pkCol] = key.slice(prefix.length);
|
||
return row;
|
||
});
|
||
// v0.3.3: 事务内合并未提交快照（update/delete/count/clear 也能看到本事务的写入）
|
||
return this.mergeTxnSnapshot(tableName, rows);
|
||
}
|
||
|
||
/**
|
||
* v0.3.3: 将事务未提交快照的变更合并到行列表（新增/更新/删除标记）。
|
||
* 幂等操作：行已是最新时不重复修改。
|
||
*/
|
||
private mergeTxnSnapshot(tableName: string, rows: Record<string, unknown>[]): Record<string, unknown>[] {
|
||
if (!this.currentTxnId || !this.txnSnapshot) return rows;
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const prefix = `${tableName}:`;
|
||
for (const [key, value] of this.txnSnapshot) {
|
||
if (!key.startsWith(prefix)) continue;
|
||
const pk = key.slice(prefix.length);
|
||
const del = (value as unknown as Record<string, unknown>).__txn_deleted;
|
||
const idx = rows.findIndex((r) => r[pkCol] === pk);
|
||
if (del) {
|
||
if (idx >= 0) rows.splice(idx, 1);
|
||
} else {
|
||
const row = { ...value, [pkCol]: pk };
|
||
if (idx >= 0) rows[idx] = row;
|
||
else rows.push(row);
|
||
}
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
private getPK(schema: TableSchema): string {
|
||
for (const [name, col] of Object.entries(schema.columns)) {
|
||
if (col.primaryKey) return name;
|
||
}
|
||
return Object.keys(schema.columns)[0];
|
||
}
|
||
|
||
private validateRow(schema: TableSchema, row: Record<string, unknown>): Record<string, unknown> {
|
||
const validated: Record<string, unknown> = {};
|
||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||
let value = row[colName];
|
||
if (value === undefined && colDef.default !== undefined) value = colDef.default;
|
||
if (colDef.required && (value === undefined || value === null)) {
|
||
throw new DatabaseError(`Column "${colName}" is required in table "${schema.name}"`, 'VALIDATION_ERROR');
|
||
}
|
||
// v0.7.4: 主键列强制非空（SQL 语义 PK 隐含 NOT NULL）——
|
||
// 此前 null/undefined 主键被 String() 化为 "null"/"undefined" 静默入库
|
||
if (colDef.primaryKey && (value === undefined || value === null)) {
|
||
throw new DatabaseError(
|
||
`Primary key column "${colName}" in table "${schema.name}" cannot be null or undefined`,
|
||
'VALIDATION_ERROR',
|
||
);
|
||
}
|
||
if (value !== undefined && value !== null) {
|
||
this.checkType(colName, colDef.type, value, colDef);
|
||
}
|
||
if (value !== undefined) validated[colName] = value;
|
||
}
|
||
return validated;
|
||
}
|
||
|
||
private checkType(colName: string, type: string, value: unknown, colDef?: ColumnDef): void {
|
||
checkFieldType('', colName, type as any, value, colDef);
|
||
}
|
||
|
||
// =======================================================================
|
||
// Schema 持久化
|
||
// =======================================================================
|
||
|
||
private async persistSchemas(): Promise<void> {
|
||
const data: Record<string, Record<string, ColumnDef>> = {};
|
||
for (const [name, schema] of this.schemas) {
|
||
data[name] = schema.columns;
|
||
}
|
||
const json = JSON.stringify(data);
|
||
const buf = new TextEncoder().encode(json).buffer;
|
||
await this.backend.write('__aria_schemas', buf);
|
||
}
|
||
|
||
private async loadSchemas(): Promise<void> {
|
||
const raw = await this.backend.read('__aria_schemas');
|
||
if (!raw) return;
|
||
|
||
try {
|
||
const json = new TextDecoder().decode(raw);
|
||
const data = JSON.parse(json) as Record<string, Record<string, ColumnDef>>;
|
||
|
||
for (const [tableName, columns] of Object.entries(data)) {
|
||
const schema: TableSchema = { name: tableName, columns };
|
||
this.schemas.set(tableName, schema);
|
||
this.tablePKs.set(tableName, this.getPK(schema));
|
||
}
|
||
} catch {
|
||
// 忽略损坏的 schema 数据
|
||
}
|
||
}
|
||
|
||
// =======================================================================
|
||
// SSTableStore 构建
|
||
// =======================================================================
|
||
|
||
/**
|
||
* 创建命名空间隔离的 SSTableStore。
|
||
*
|
||
* 主 LSM 与每个二级索引 LSM 各持有独立实例：
|
||
* - 文件 key 前缀隔离（sst_ / sst_idx_${table}_${col}_）
|
||
* - 元数据 key 隔离（__aria_lsm_meta / __aria_lsm_meta_${ns}）
|
||
* - id 序列独立（避免 v0.2.4 共享 id 空间导致的文件互相覆盖）
|
||
*/
|
||
private createSSTableStore(ns: string): SSTableStore {
|
||
const filePrefix = ns === 'main' ? 'sst_' : `sst_${ns}_`;
|
||
const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`;
|
||
let seq = 0;
|
||
let seqLoaded = false;
|
||
|
||
// v0.4.5: 页面化物理存储（OPFS 后端默认启用）— SSTable 存为 4KB 页面，BufferPool 缓存
|
||
const usePages = this.isPageStorage();
|
||
const pageStore = usePages ? new PageSSTableStore(this.fileManager, this.bufferPool) : null;
|
||
|
||
const encodeText = (text: string): ArrayBuffer => {
|
||
return new TextEncoder().encode(text).buffer;
|
||
};
|
||
|
||
const readMetaList = async (): Promise<SSTableMeta[]> => {
|
||
const raw = await this.backend.read(META_KEY);
|
||
if (!raw) return [];
|
||
try {
|
||
return JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[];
|
||
} catch {
|
||
return [];
|
||
}
|
||
};
|
||
|
||
return {
|
||
save: async (id, data) => {
|
||
if (pageStore) {
|
||
// 页面化：切页写入 BufferPool 并逐页落盘（save 语义 = 已持久化）
|
||
await pageStore.save(id, data);
|
||
return;
|
||
}
|
||
let buf: ArrayBuffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
|
||
// 压缩（若启用）— 加密由 EncryptedBackend 在 backend 层透明处理（v0.4.5）
|
||
if (this.config.compression) {
|
||
const compressed = compressLZ4(new Uint8Array(buf));
|
||
buf = compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer;
|
||
}
|
||
await this.backend.write(`${filePrefix}${id}`, buf);
|
||
},
|
||
load: async (id) => {
|
||
// 页面化读取：meta 有 pageIds → 页面拼接；无（旧数据）→ 整 value
|
||
if (pageStore) {
|
||
const metas = await readMetaList();
|
||
const meta = metas.find((m) => m.id === id);
|
||
if (meta && meta.pageIds && meta.pageIds.length > 0) {
|
||
return pageStore.load(id, meta.pageIds, meta.totalSize);
|
||
}
|
||
}
|
||
const raw = await this.backend.read(`${filePrefix}${id}`);
|
||
if (!raw) return null;
|
||
let buf = new Uint8Array(raw);
|
||
// 解压（若启用）— 解密由 EncryptedBackend 在 backend 层透明处理（v0.4.5）
|
||
if (this.config.compression) {
|
||
// v0.4.5: 压缩流自带原始大小头，无需外部估算
|
||
buf = decompressLZ4(buf) as Uint8Array<ArrayBuffer>;
|
||
}
|
||
return buf;
|
||
},
|
||
delete: async (id) => {
|
||
if (pageStore) {
|
||
const metas = await readMetaList();
|
||
const meta = metas.find((m) => m.id === id);
|
||
if (meta && meta.pageIds && meta.pageIds.length > 0) {
|
||
await pageStore.delete(id, meta.pageIds);
|
||
}
|
||
}
|
||
await this.backend.delete(`${filePrefix}${id}`);
|
||
},
|
||
allocateId: async () => {
|
||
// 从本命名空间的 meta 恢复 id 序列，保证单调递增且不与其他 LSM 冲突
|
||
if (!seqLoaded) {
|
||
const metas = await readMetaList();
|
||
seq = metas.reduce((m, x) => Math.max(m, x.id), 0);
|
||
seqLoaded = true;
|
||
}
|
||
return ++seq;
|
||
},
|
||
listMeta: readMetaList,
|
||
saveMeta: async (meta) => {
|
||
const list = await readMetaList();
|
||
// v0.4.5: 页面化时把页面 ID 列表注入 meta（save 后、saveMeta 前由 LSM 顺序调用）
|
||
const pageIds = pageStore?.getPageIds(meta.id);
|
||
const metaWithPages = pageIds && pageIds.length > 0 ? { ...meta, pageIds } : meta;
|
||
// 更新或添加
|
||
const idx = list.findIndex((m) => m.id === meta.id);
|
||
if (idx >= 0) list[idx] = metaWithPages;
|
||
else list.push(metaWithPages);
|
||
await this.backend.write(META_KEY, encodeText(JSON.stringify(list)));
|
||
},
|
||
deleteMeta: async (id) => {
|
||
const list = await readMetaList();
|
||
const filtered = list.filter((m) => m.id !== id);
|
||
await this.backend.write(META_KEY, encodeText(JSON.stringify(filtered)));
|
||
},
|
||
};
|
||
}
|
||
|
||
/** v0.4.5: 是否启用页面化物理存储（默认 OPFS / KVStore 后端启用，显式配置可覆盖） */
|
||
private isPageStorage(): boolean {
|
||
if (this.config.pageStorage === true) return true;
|
||
if (this.config.pageStorage === false) return false;
|
||
// v0.6.1: kv 后端同样页面化 — 整 value SSTable（4MB）超过 1MB 页面缓存时
|
||
// 每次读取全量重载；拆 4KB 页面后缓存按页命中，大数据量读放大消除
|
||
return this.config.storageBackend === 'opfs' || this.config.storageBackend === 'kv';
|
||
}
|
||
|
||
// =======================================================================
|
||
// WAL 恢复
|
||
// =======================================================================
|
||
|
||
private applyWALRecord(record: WALRecord): void {
|
||
switch (record.type) {
|
||
case WALRecordType.INSERT:
|
||
case WALRecordType.UPDATE:
|
||
if (record.data) {
|
||
this.lsm.put(`${record.tableName}:${record.key}`, record.data);
|
||
}
|
||
break;
|
||
case WALRecordType.DELETE:
|
||
this.lsm.delete(`${record.tableName}:${record.key}`);
|
||
break;
|
||
case WALRecordType.CREATE_TABLE:
|
||
if (record.data?.schema) {
|
||
try {
|
||
const s = JSON.parse(record.data.schema as string) as TableSchema;
|
||
if (!this.schemas.has(s.name)) {
|
||
this.schemas.set(s.name, s);
|
||
this.tablePKs.set(s.name, this.getPK(s));
|
||
}
|
||
} catch { /* skip */ }
|
||
}
|
||
break;
|
||
case WALRecordType.COMMIT:
|
||
case WALRecordType.ROLLBACK:
|
||
case WALRecordType.BEGIN:
|
||
break;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.3.3: DROP_TABLE 恢复 — 删除 schema 并清除主 LSM 中该表的所有残留数据。
|
||
*
|
||
* 此前 DROP_TABLE 在恢复时被忽略，而 CREATE_TABLE 回放会重建 schema，
|
||
* 导致崩溃后"已删除的表和数据复活"（实证 P0 bug）。
|
||
*/
|
||
private async applyDropTableRecovery(tableName: string): Promise<void> {
|
||
if (!tableName) return;
|
||
// v0.4.2-fix: 清理该表二级索引（崩溃恢复路径同样不留孤儿索引）
|
||
await this.cleanupTableIndexes(tableName);
|
||
this.schemas.delete(tableName);
|
||
this.tablePKs.delete(tableName);
|
||
|
||
// 清除主 LSM 中该表前缀的所有数据（含 SSTable 中的旧数据）
|
||
const prefix = `${tableName}:`;
|
||
const endKey = `${prefix}\uffff`;
|
||
await this.lsm.prefetchRange(prefix, endKey);
|
||
const entries = await this.lsm.rangeScan(prefix, endKey);
|
||
for (const [key] of entries) {
|
||
this.lsm.delete(key);
|
||
}
|
||
}
|
||
|
||
// =======================================================================
|
||
// 二级索引
|
||
// =======================================================================
|
||
|
||
/** v0.6.2: 表中有 unique 约束且索引 LSM 已建的列（唯一性检查范围） */
|
||
private uniqueColumns(tableName: string, schema: TableSchema): string[] {
|
||
const cols: string[] = [];
|
||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||
if (!colDef.unique) continue;
|
||
if (this.secondaryIndexes.has(`${tableName}:idx:${colName}`)) cols.push(colName);
|
||
}
|
||
return cols;
|
||
}
|
||
|
||
/**
|
||
* 唯一性检查。
|
||
*
|
||
* v0.8.0: 由 `checkUniqueSync` 改名并改为 async —— 此前命名为 "Sync" 是因为它
|
||
* 依赖"批次级 prefetchPrefixRanges 之后索引数据已在缓存中"这一约定。现在
|
||
* LSM 读取自洽（未命中即回源），因此这里可以、也必须 await。
|
||
* 索引不含 null 条目（null 值不受唯一约束，与 MemoryEngine 语义一致）。
|
||
* @param currentPk 当前行主键（更新路径用于排除自身旧索引条目；插入路径无自身条目）
|
||
*/
|
||
private async checkUnique(
|
||
tableName: string,
|
||
uniqueCols: string[],
|
||
row: Record<string, unknown>,
|
||
currentPk: string,
|
||
): Promise<void> {
|
||
for (const colName of uniqueCols) {
|
||
const val = row[colName];
|
||
if (val === undefined || val === null) continue;
|
||
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`);
|
||
if (!idxLsm) continue;
|
||
const prefix = `${String(val)}:`;
|
||
const entries = await idxLsm.rangeScan(prefix, `${prefix}\uffff`);
|
||
for (const [, entry] of entries) {
|
||
const pk = (entry as unknown as { pk?: string }).pk;
|
||
if (pk !== undefined && pk !== currentPk) {
|
||
throw new DatabaseError(
|
||
`Unique constraint violation on column "${colName}" in table "${tableName}"`,
|
||
'UNIQUE_VIOLATION',
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/** 更新行的二级索引条目 */
|
||
private updateSecondaryIndexes(
|
||
tableName: string, pkValue: string,
|
||
newRow: Record<string, unknown> | null,
|
||
oldRow: Record<string, unknown> | null,
|
||
): void {
|
||
const schema = this.schemas.get(tableName);
|
||
if (!schema) return;
|
||
|
||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||
// v0.3.3: 主键列不建冗余二级索引（主 LSM 即 PK 索引）
|
||
if (!colDef.index && !colDef.unique) continue;
|
||
const idxKey = `${tableName}:idx:${colName}`;
|
||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||
if (!idxLsm) continue;
|
||
|
||
// 删除旧值
|
||
if (oldRow) {
|
||
const oldVal = oldRow[colName];
|
||
if (oldVal !== undefined && oldVal !== null) {
|
||
idxLsm.delete(`${String(oldVal)}:${pkValue}`);
|
||
}
|
||
}
|
||
|
||
// 插入新值
|
||
if (newRow) {
|
||
const newVal = newRow[colName];
|
||
if (newVal !== undefined && newVal !== null) {
|
||
idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue });
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/** 通过二级索引快速查找 */
|
||
private async tryIndexLookup(
|
||
tableName: string,
|
||
query: QueryPlan,
|
||
): Promise<Record<string, unknown>[] | null> {
|
||
if (!query.where) return null;
|
||
const schema = this.schemas.get(tableName);
|
||
if (!schema) return null;
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
|
||
// v0.7.3: 递归展开 $and 中的等值条件 —— 此前仅顶层键，
|
||
// `WHERE a AND b`（解析为顶层 $and）永远全表扫描，索引形同虚设。
|
||
// $or/$not 语义不适用单索引下推，保守跳过。命中索引后 find 仍以
|
||
// 全条件 matchWhere 过滤（子集语义安全）。
|
||
const flat: [string, unknown][] = [];
|
||
const collect = (w: WhereCondition): void => {
|
||
for (const [k, v] of Object.entries(w)) {
|
||
if (k === '$and') {
|
||
for (const sub of (v as WhereCondition[])) collect(sub);
|
||
continue;
|
||
}
|
||
if (k === '$or' || k === '$not' || k === '$exists') continue;
|
||
flat.push([k, v]);
|
||
}
|
||
};
|
||
collect(query.where);
|
||
|
||
for (const [col, condition] of flat) {
|
||
|
||
const colDef = schema.columns[col];
|
||
const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey);
|
||
if (!hasIndex && col !== pkCol) continue;
|
||
|
||
// PK 等值 → 主 LSM 精确查找
|
||
if (col === pkCol) {
|
||
if (typeof condition !== 'object' || condition === null) {
|
||
const key = `${tableName}:${condition}`;
|
||
await this.lsm.prefetchKeys([key]);
|
||
const value = await this.lsm.get(key);
|
||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||
}
|
||
const cond = condition as Record<string, unknown>;
|
||
if ('$eq' in cond) {
|
||
const key = `${tableName}:${cond.$eq}`;
|
||
await this.lsm.prefetchKeys([key]);
|
||
const value = await this.lsm.get(key);
|
||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||
}
|
||
// v0.3.3: PK $in → 主 LSM 多次精确查找（替代冗余 PK 二级索引）
|
||
if ('$in' in cond && Array.isArray(cond.$in)) {
|
||
const keys = cond.$in.map((v) => `${tableName}:${v}`);
|
||
await this.lsm.prefetchKeys(keys);
|
||
const rows: Record<string, unknown>[] = [];
|
||
const seen = new Set<string>(); // v0.4.1: IN 子查询可能含重复值，按 pk 去重
|
||
for (const v of cond.$in) {
|
||
const pk = String(v);
|
||
if (seen.has(pk)) continue;
|
||
const value = await this.lsm.get(`${tableName}:${pk}`);
|
||
if (value) { seen.add(pk); rows.push({ ...value, [pkCol]: pk }); }
|
||
}
|
||
return rows;
|
||
}
|
||
// v0.3.3: PK 范围查询 → 主 LSM 前缀扫描 + 条件过滤（修复字符串算术 bug）
|
||
if ('$gt' in cond || '$gte' in cond || '$lt' in cond || '$lte' in cond) {
|
||
const prefix = `${tableName}:`;
|
||
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
|
||
const entries = await this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||
const rows: Record<string, unknown>[] = [];
|
||
for (const [key, value] of entries) {
|
||
const candidate = { ...value, [pkCol]: key.slice(prefix.length) };
|
||
if (matchWhere(candidate, { [pkCol]: condition })) rows.push(candidate);
|
||
}
|
||
return rows;
|
||
}
|
||
}
|
||
|
||
// 二级索引查找
|
||
const idxKey = `${tableName}:idx:${col}`;
|
||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||
if (!idxLsm) continue;
|
||
|
||
// $eq → 精确查找
|
||
if (typeof condition !== 'object' || condition === null) {
|
||
// v0.6.2-fix(P1): IS NULL（条件为 null）不走索引 —— 索引不含 null 条目，
|
||
// String(null)="null" 查找返回空并短路全表 → 索引列 IS NULL 恒空
|
||
if (condition === null) continue;
|
||
return this.indexScanToRows(tableName, pkCol, idxLsm, String(condition), String(condition));
|
||
}
|
||
const c = condition as Record<string, unknown>;
|
||
if ('$eq' in c) {
|
||
// v0.6.2-fix(P1): 同上，$eq: null（IS NULL）不走索引
|
||
if (c.$eq === null) continue;
|
||
const v = String(c.$eq);
|
||
return this.indexScanToRows(tableName, pkCol, idxLsm, v, v);
|
||
}
|
||
// $in → 多次精确查找
|
||
if ('$in' in c && Array.isArray(c.$in)) {
|
||
// v0.6.2-fix(P1): IN 列表含 null 不走索引（索引不含 null 条目，会漏匹配 null 行）
|
||
if (c.$in.some((v) => v === null)) continue;
|
||
// v0.7.3-perf: 批级预加载全部值的索引范围 + 主表行（各一次 drainChain）——
|
||
// 此前逐值 indexScanToRows：每个值一次 prefetchRange + prefetchKeys，
|
||
// 后台 compaction 长耗时时 N 倍放大（与 v0.6.1 修的 insert 批量预加载
|
||
// 性能悬崖同类）。批级预加载后循环内同步 rangeScan/get。
|
||
const values = c.$in.map((v) => String(v));
|
||
await idxLsm.prefetchPrefixRanges(values.map((v): [string, string] => [v, `${v}\uffff`]));
|
||
const results: Record<string, unknown>[] = [];
|
||
const seenPks = new Set<string>(); // v0.4.1: IN 值可能重复，按 pk 去重
|
||
const pks: string[] = [];
|
||
for (const val of values) {
|
||
const entries = await idxLsm.rangeScan(val, `${val}\uffff`);
|
||
for (const [, idxEntry] of entries) {
|
||
const pk = (idxEntry as { pk?: string }).pk;
|
||
if (pk && !seenPks.has(pk)) {
|
||
seenPks.add(pk);
|
||
pks.push(pk);
|
||
}
|
||
}
|
||
}
|
||
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
|
||
for (const pk of pks) {
|
||
const row = await this.lsm.get(`${tableName}:${pk}`);
|
||
if (row) results.push({ ...row, [pkCol]: pk });
|
||
}
|
||
return results;
|
||
}
|
||
// $gt / $gte / $lt / $lte → 范围扫描
|
||
if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) {
|
||
// v0.6.2-fix(P1): 此前用 Number(v)±1 构造边界 key —— 小数数值
|
||
// （$gt:2 → "3:"，漏 2.5）与字符串（"NaN:" 前缀错位，数字/大写开头值被漏）
|
||
// 静默丢数据。改为全索引扫描 + 行级 matchWhere 过滤（与主键范围路径同方案），
|
||
// 边界语义与 where-matcher 完全一致。
|
||
const rows = await this.indexScanToRows(tableName, pkCol, idxLsm, '', '\uffff');
|
||
return rows.filter((row) => matchWhere(row, { [col]: condition }));
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
/** 从索引扫描结果恢复完整行 */
|
||
private async indexScanToRows(
|
||
tableName: string, pkCol: string, idxLsm: LSM,
|
||
startKey: string, endKey: string,
|
||
): Promise<Record<string, unknown>[]> {
|
||
// 使用前缀扫描：endKey 需要包含 \uffff 以匹配所有带后缀的 key
|
||
const actualEndKey = endKey.includes('\uffff') ? endKey : `${endKey}\uffff`;
|
||
// 预加载索引 LSM 与主 LSM 涉及的 SSTable
|
||
await idxLsm.prefetchRange(startKey, actualEndKey);
|
||
const entries = await idxLsm.rangeScan(startKey, actualEndKey);
|
||
const pks: string[] = [];
|
||
for (const [, idxEntry] of entries) {
|
||
const pk = (idxEntry as any).pk as string;
|
||
if (pk) pks.push(pk);
|
||
}
|
||
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
|
||
const rows: Record<string, unknown>[] = [];
|
||
for (const pk of pks) {
|
||
const row = await this.lsm.get(`${tableName}:${pk}`);
|
||
if (row) rows.push({ ...row, [pkCol]: pk });
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
// =======================================================================
|
||
// 辅助
|
||
// =======================================================================
|
||
|
||
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
|
||
private tryGC(): void {
|
||
this.gcCounter++;
|
||
if (this.gcCounter >= 10) {
|
||
this.mvcc.gc(100);
|
||
this.gcCounter = 0;
|
||
}
|
||
}
|
||
|
||
/** 回收主 LSM 与所有二级索引 LSM 的临时缓存超限 */
|
||
private trimAllCaches(): void {
|
||
this.lsm.trimCache();
|
||
for (const idxLsm of this.secondaryIndexes.values()) {
|
||
idxLsm.trimCache();
|
||
}
|
||
}
|
||
|
||
/** 检查内存预算，超出时强制 flush + GC */
|
||
private checkMemoryBudget(): void {
|
||
const maxBytes = this.config.maxMemoryMB * 1024 * 1024;
|
||
const used = this.lsm.getEstimatedMemory();
|
||
if (used > maxBytes) {
|
||
this.lsm.flush().catch(() => {});
|
||
this.mvcc.gc(50);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* ANALYZE: 收集表统计信息
|
||
* 返回行数、平均行大小、索引深度等
|
||
*/
|
||
async analyzeTable(tableName: string): Promise<Record<string, unknown>> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
const rows = await this.getAllRows(tableName);
|
||
// v0.7.3: 统计汇总主 LSM + 该表全部二级索引 LSM —— 此前只统计主 LSM，
|
||
// 表带多个索引时索引深度/SSTable 数量严重低估
|
||
let sstableCount = this.lsm.getStats().sstableCount;
|
||
let memtableSize = this.lsm.getStats().memtableSize;
|
||
let indexDepth = this.lsm.getStats().levelCounts.filter((c: number) => c > 0).length;
|
||
for (const [idxKey, idxLsm] of this.secondaryIndexes) {
|
||
if (!idxKey.startsWith(`${tableName}:idx:`)) continue;
|
||
const s = idxLsm.getStats();
|
||
sstableCount += s.sstableCount;
|
||
memtableSize += s.memtableSize;
|
||
indexDepth = Math.max(indexDepth, s.levelCounts.filter((c: number) => c > 0).length);
|
||
}
|
||
const stats: Record<string, unknown> = {
|
||
table: tableName,
|
||
rowCount: rows.length,
|
||
avgRowSize: rows.length > 0
|
||
? Math.round(rows.reduce((s, r) => s + JSON.stringify(r).length, 0) / rows.length)
|
||
: 0,
|
||
indexDepth,
|
||
sstableCount,
|
||
memtableSize,
|
||
estimatedMemory: this.lsm.getEstimatedMemory(),
|
||
};
|
||
|
||
// 列基数统计
|
||
const schema = this.schemas.get(tableName);
|
||
if (schema && rows.length > 0) {
|
||
const columnStats: Record<string, unknown> = {};
|
||
for (const colName of Object.keys(schema.columns)) {
|
||
const values = new Set(rows.map((r) => String(r[colName])));
|
||
columnStats[colName] = { distinctValues: values.size };
|
||
}
|
||
stats.columnStats = columnStats;
|
||
}
|
||
|
||
return stats;
|
||
}
|
||
|
||
/**
|
||
* REINDEX: 重建指定表的所有二级索引
|
||
*/
|
||
async reindexTable(tableName: string): Promise<number> {
|
||
this.ensureOpen();
|
||
this.ensureTable(tableName);
|
||
return this.reindexTableInternal(tableName);
|
||
}
|
||
|
||
/** v0.4.2-fix: 重建索引内部实现（不校验 opened，供 open 恢复流程调用） */
|
||
private async reindexTableInternal(tableName: string): Promise<number> {
|
||
const schema = this.schemas.get(tableName);
|
||
if (!schema) return 0;
|
||
let rebuiltCount = 0;
|
||
|
||
// v0.7.4-perf: 单次全表扫描重建全部索引列 —— 此前每个索引列各做一次
|
||
// getAllRows（N 列 × M 行全表扫描 + 每次 prefetchRange drainChain），
|
||
// 多索引大表 REINDEX/崩溃恢复按索引列数线性放大
|
||
const pkCol = this.tablePKs.get(tableName)!;
|
||
const idxCols = Object.entries(schema.columns).filter(
|
||
([, colDef]) => colDef.index || colDef.unique,
|
||
);
|
||
if (idxCols.length === 0) return 0;
|
||
|
||
const rows = await this.getAllRows(tableName);
|
||
for (const [colName] of idxCols) {
|
||
const idxKey = `${tableName}:idx:${colName}`;
|
||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||
if (!idxLsm) continue;
|
||
|
||
// 清空旧索引
|
||
await idxLsm.clear();
|
||
rebuiltCount++;
|
||
|
||
// 从主 LSM 重建索引
|
||
for (const row of rows) {
|
||
const val = row[colName];
|
||
if (val !== undefined && val !== null) {
|
||
idxLsm.put(`${String(val)}:${row[pkCol]}`, { pk: row[pkCol] });
|
||
}
|
||
}
|
||
}
|
||
return rebuiltCount;
|
||
}
|
||
|
||
/**
|
||
* VACUUM: 压缩 LSM + 清理碎片
|
||
*/
|
||
async vacuum(): Promise<{ compactedLevels: number; gcVersions: number }> {
|
||
this.ensureOpen();
|
||
// 强制 flush memtable
|
||
await this.lsm.flush();
|
||
// 压缩各层级
|
||
for (let level = 0; level < 6; level++) {
|
||
if (this.lsm.getStats().levelCounts[level] >= 2) {
|
||
await this.lsm.compactLevel(level);
|
||
}
|
||
}
|
||
// GC MVCC 版本（保留最新 10 个）
|
||
const beforeGC = this.mvcc.getGlobalLSN();
|
||
this.mvcc.gc(10);
|
||
return { compactedLevels: 6, gcVersions: beforeGC };
|
||
}
|
||
|
||
private ensureOpen(): void {
|
||
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||
}
|
||
|
||
/** v0.4.2-fix: Aria 事务中 DDL 显式拒绝（结构变更无法通过行快照回滚） */
|
||
private ensureNoDDLInTransaction(op: string): void {
|
||
if (this.currentTxnId) {
|
||
throw new DatabaseError(
|
||
`${op} is not supported inside a transaction (AriaEngine DDL is not transactional)`,
|
||
'NOT_SUPPORTED',
|
||
);
|
||
}
|
||
}
|
||
|
||
private ensureTable(tableName: string): void {
|
||
if (!this.schemas.has(tableName)) {
|
||
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
|
||
}
|
||
}
|
||
}
|