release: v0.4.2 — 生产就绪与崩溃自愈 + 问题清单修复 + 版本迭代
CI / test (18.x) (push) Successful in 10m7s
CI / test (20.x) (push) Successful in 10m6s
CI / test (22.x) (push) Successful in 10m2s
CI / test (24.x) (push) Successful in 10m0s

This commit is contained in:
thzxx
2026-08-09 19:15:37 +08:00
parent a1e4f5071c
commit 22b0b1fad4
34 changed files with 6626 additions and 565 deletions
+312 -39
View File
@@ -70,6 +70,22 @@ export class AriaEngine implements IStorageEngine {
async open(dbName: string, _version: number): Promise<void> {
if (this.opened) return;
// v0.4.2-fix: 引擎内部错误统一包装为 DatabaseErrorARIA_OPEN_ERROR),
// 应用层可拿到 code 分类处理,不再抛出原生 RangeError/TypeError
try {
await this.openInternal(dbName);
} catch (error) {
if (error instanceof DatabaseError) throw error;
throw new DatabaseError(
`Failed to open AriaEngine database "${dbName}"`,
'ARIA_OPEN_ERROR',
error,
);
}
}
/** open 内部实现(错误包装在 open 外层) */
private async openInternal(dbName: string): Promise<void> {
this.dbName = dbName;
// 1. 存储后端
@@ -105,20 +121,30 @@ export class AriaEngine implements IStorageEngine {
this.wal = new WAL(
{
append: async (data) => {
// Store each record as a separate numbered key
// v0.4.2-fix: 记录写入与 count 计数在同一底层事务中原子提交,
// 中断时整体回滚,杜绝"记录在、计数丢"导致恢复漏读的丢数据问题
const idx = await this.getWALCount();
const slice = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
const copy = slice.buffer.slice(slice.byteOffset, slice.byteOffset + slice.byteLength) as ArrayBuffer;
await this.backend.write(`__wal_${idx}`, copy);
await this.setWALCount(idx + 1);
await this.backend.writeMany({
[`__wal_${idx}`]: copy,
__wal_count: new TextEncoder().encode(String(idx + 1)).buffer,
});
},
readAll: async () => {
const count = await this.getWALCount();
if (count === 0) return new Uint8Array(0);
// Read all records and concatenate
// v0.4.2-fix: 不依赖 count 计数,直接扫描全部 __wal_* 键,
// 避免 count 与实际记录不一致时漏读(与 checkpoint/并发写入竞态无关)
const keys = (await this.backend.listKeys())
.filter((k) => k.startsWith('__wal_') && k !== '__wal_count')
.sort((a, b) => {
const na = parseInt(a.slice('__wal_'.length), 10);
const nb = parseInt(b.slice('__wal_'.length), 10);
return (isNaN(na) ? 0 : na) - (isNaN(nb) ? 0 : nb);
});
if (keys.length === 0) return new Uint8Array(0);
const chunks: Uint8Array[] = [];
for (let i = 0; i < count; i++) {
const d = await this.backend.read(`__wal_${i}`);
for (const key of keys) {
const d = await this.backend.read(key);
if (d) chunks.push(new Uint8Array(d));
}
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
@@ -128,15 +154,17 @@ export class AriaEngine implements IStorageEngine {
return combined;
},
truncate: async () => {
const count = await this.getWALCount();
for (let i = 0; i < count; i++) {
await this.backend.delete(`__wal_${i}`);
}
// v0.4.2-fix: 扫描删除全部 WAL 记录键 + count 键(单事务原子清理)
const keys = (await this.backend.listKeys())
.filter((k) => k.startsWith('__wal_'));
await this.backend.deleteMany(keys);
await this.setWALCount(0);
},
exists: async () => {
const count = await this.getWALCount();
return count > 0;
// v0.4.2-fix: 与 readAll 一致按 key 扫描判断(count 可能因崩溃截断而滞后)
const keys = (await this.backend.listKeys())
.filter((k) => k.startsWith('__wal_') && k !== '__wal_count');
return keys.length > 0;
},
},
this.config.walEnabled,
@@ -146,6 +174,31 @@ export class AriaEngine implements IStorageEngine {
// 5. 恢复 Schema
await this.loadSchemas();
// v0.4.2-fix: 为 schema 中带 index/unique 标记的列重建二级索引 LSM。
// 此前重开只恢复 schema 不恢复索引 LSM → 索引查询静默回退全表、
// createIndex 因 colDef 已有标记直接 return → 索引永久缺失。
// 索引数据已持久化在独立命名空间(sst_idx_* / meta),init() 直接加载。
for (const [tableName, schema] of this.schemas) {
const pkCol = this.tablePKs.get(tableName)!;
for (const [colName, colDef] of Object.entries(schema.columns)) {
if ((colDef.index || colDef.unique) && colName !== pkCol) {
const idxKey = `${tableName}:idx:${colName}`;
if (!this.secondaryIndexes.has(idxKey)) {
const idxLsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore: this.createSSTableStore(`idx_${tableName}_${colName}`),
});
await idxLsm.init();
this.secondaryIndexes.set(idxKey, idxLsm);
}
}
}
}
// 6. 初始化 LSM(加载 SSTable 元数据)
await this.lsm.init();
@@ -175,12 +228,31 @@ export class AriaEngine implements IStorageEngine {
if (allRecords.length > 0) {
await this.lsm.flush();
await this.wal.checkpoint();
// v0.4.2-fix: WAL 回放只更新主 LSM,二级索引 LSM 未同步 →
// 崩溃前最后一批写入的索引缺失,重开时索引查询丢行。
// 恢复后全量重建所有表的二级索引(幂等)。
for (const tableName of this.schemas.keys()) {
await this.reindexTableInternal(tableName);
}
}
// 8. Checkpoint Manager(接入 WAL 大小阈值)
// v0.4.2-fix: 事务活跃时 checkpoint 不得截断 WAL —
// 否则 BEGIN/INSERT 记录被截断,COMMIT 后崩溃恢复丢失整个事务数据
this.checkpointManager = new CheckpointManager(
this.lsm,
this.wal,
{
checkpoint: async () => {
if (this.currentTxnId) return;
await this.wal.checkpoint();
},
flush: async () => {
if (this.currentTxnId) return;
await this.wal.flush();
},
getBufferedBytes: () => this.wal.getBufferedBytes(),
getBufferedCount: () => this.wal.getBufferedCount(),
} as unknown as WAL,
{ flushAll: async () => { await this.lsm.flush(); } } as any,
this.config.checkpointInterval,
this.config.walSizeThreshold,
@@ -193,12 +265,51 @@ export class AriaEngine implements IStorageEngine {
if (!this.opened) return;
await this.persistSchemas();
await this.lsm.flush();
// v0.4.2-fix: 同步落盘全部二级索引 LSM — 此前只 flush 主 LSM
// 优雅关闭后索引 memtable 未落盘 → 重开索引为空 → 索引查询返回空结果
for (const idxLsm of this.secondaryIndexes.values()) {
await idxLsm.flush();
}
await this.wal.flush();
// v0.4.2-fix: close 前 checkpoint(截断 WAL)—
// 此前只 flush 不截断,下次打开会重放全部历史 WAL 记录(含已落盘 SSTable 的数据),
// 重复解析/重复 put 拖慢启动,并与恢复后 flush+checkpoint 竞争放大丢数据
await this.wal.checkpoint();
await this.backend.close();
// v0.4.2-fix: 清空运行期状态(此前 close 后 mvcc/txn 残留,
// 重开时 beginTransaction 报 TX_ACTIVE 或读到陈旧快照)
this.schemas.clear();
this.tablePKs.clear();
this.secondaryIndexes.clear();
this.mvcc = new MVCCManager();
this.currentTxnId = null;
this.txnSnapshot = null;
this.savepoints.clear();
this.opCounter = 0;
this.opened = false;
}
/**
* v0.4.2-fix: 崩溃恢复/自愈 — 校验并移除损坏 SSTable、截断 WAL、重建二级索引。
* 应用层检测到异常后调用,无需删库重建。
*/
async repair(): Promise<void> {
this.ensureOpen();
// 1. 校验全部 SSTable,移除残缺项(打开时已做一次,此处兜底运行期损坏)
const removed = await this.lsm.validateAll();
// 2. 将 WAL 残留数据落盘并截断,避免无限重放
await this.lsm.flush();
await this.wal.checkpoint();
// 3. 重建所有表的二级索引(修复索引与主数据不一致)
for (const tableName of this.schemas.keys()) {
await this.reindexTable(tableName);
}
if (removed > 0) {
// eslint-disable-next-line no-console
console.warn(`[AriaEngine] repair: removed ${removed} corrupted SSTable(s)`);
}
}
/**
* v0.4.1: 重置数据库 — 清空全部数据与表结构(演示页刷新/重新初始化用)。
* 清空存储后端、LSM、WAL、MVCC 与二级索引,后续可继续使用本实例。
@@ -224,12 +335,26 @@ export class AriaEngine implements IStorageEngine {
isOpen(): boolean { return this.opened; }
// ---- v0.4.2-fix: 库内元数据(迁移版本持久化用) ----
async getMeta(key: string): Promise<string | null> {
const raw = await this.backend.read(`__meta_${key}`);
return raw ? new TextDecoder().decode(raw) : null;
}
async setMeta(key: string, value: string): Promise<void> {
await this.backend.write(`__meta_${key}`, new TextEncoder().encode(value).buffer);
}
// =======================================================================
// 表管理
// =======================================================================
async createTable(schema: TableSchema): Promise<void> {
this.ensureOpen();
// v0.4.2-fix: Aria 事务中 DDL 显式拒绝(事务快照只覆盖行数据,
// 结构变更无法回滚;Memory/IndexedDB 引擎快照可回滚,行为不一致 → 明确报错而非静默)
this.ensureNoDDLInTransaction('CREATE TABLE');
if (this.schemas.has(schema.name)) {
throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS');
}
@@ -270,6 +395,7 @@ export class AriaEngine implements IStorageEngine {
async dropTable(tableName: string): Promise<void> {
this.ensureOpen();
this.ensureNoDDLInTransaction('DROP TABLE');
this.ensureTable(tableName);
// 删除表中所有行
@@ -279,6 +405,10 @@ export class AriaEngine implements IStorageEngine {
this.lsm.delete(`${tableName}:${row[pkCol]}`);
}
// v0.4.2-fix: 清理该表的全部二级索引 LSM 与持久化文件 —
// 此前残留孤儿索引,重建同名表后旧索引数据污染新表(索引查询返回错误行)
await this.cleanupTableIndexes(tableName);
this.schemas.delete(tableName);
this.tablePKs.delete(tableName);
await this.persistSchemas();
@@ -291,6 +421,25 @@ export class AriaEngine implements IStorageEngine {
});
}
/**
* v0.4.2-fix: 清理指定表的全部二级索引 LSM(内存 + 存储文件 + meta)。
* dropTable / DROP_TABLE 恢复 / alterTable DROP 索引列 共用。
*/
private async cleanupTableIndexes(tableName: string): Promise<void> {
const prefix = `${tableName}:idx:`;
const toDelete: string[] = [];
for (const [idxKey, idxLsm] of this.secondaryIndexes) {
if (!idxKey.startsWith(prefix)) continue;
toDelete.push(idxKey);
try {
await idxLsm.clear();
} catch { /* 清理失败不阻塞 */ }
}
for (const idxKey of toDelete) {
this.secondaryIndexes.delete(idxKey);
}
}
async hasTable(tableName: string): Promise<boolean> {
return this.schemas.has(tableName);
}
@@ -421,6 +570,8 @@ export class AriaEngine implements IStorageEngine {
let count = 0;
// v0.3.1: 批量 WAL 写入(组提交)
const walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[] = [];
// v0.4.2-fix: ON UPDATE 级联环路保护
const visited = new Set<string>();
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName)!;
@@ -430,24 +581,48 @@ export class AriaEngine implements IStorageEngine {
const updated = { ...row, ...updates };
this.validateRow(schema, updated);
// v0.4.2-fix: 支持更新主键 — 删除旧键 + 落新键 + WAL 两条记录
const newPk = String(updated[pkCol]);
const pkChanged = newPk !== String(row[pkCol]);
if (pkChanged) {
// ON UPDATE 外键级联(RESTRICT 抛错 / CASCADE / SET NULL
await this.applyForeignKeyUpdateRules(
tableName, String(row[pkCol]), newPk, walRecords, visited,
);
}
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(key, updated);
this.mvcc.writeVersion(tableName, String(row[pkCol]), updated, this.currentTxnId);
if (pkChanged) {
this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record<string, unknown>);
this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId);
}
this.txnSnapshot.set(`${tableName}:${newPk}`, updated);
this.mvcc.writeVersion(tableName, newPk, updated, this.currentTxnId);
} else {
this.lsm.put(key, updated);
if (pkChanged) this.lsm.delete(key);
this.lsm.put(`${tableName}:${newPk}`, updated);
}
count++;
if (pkChanged) {
walRecords.push({
type: WALRecordType.DELETE,
txnId: this.currentTxnId ?? 0,
tableName,
key: String(row[pkCol]),
});
}
walRecords.push({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName,
key: String(row[pkCol]),
key: newPk,
data: updated,
});
// 更新二级索引
this.updateSecondaryIndexes(tableName, String(row[pkCol]), updated, row);
// 更新二级索引(主键变更时旧索引条目一并清理)
this.updateSecondaryIndexes(tableName, newPk, updated, pkChanged ? row : null);
}
}
@@ -457,6 +632,74 @@ export class AriaEngine implements IStorageEngine {
await this.checkpointManager.tick();
this.trimAllCaches();
return count;
}
/**
* v0.4.2-fix: ON UPDATE 外键级联 — 主键 oldPk → newPk 时处理引用表。
* RESTRICT 抛错 / CASCADE 更新 FK / SET NULL 置空(含索引与 WAL 记录)。
* 两阶段:先全量 RESTRICT 检查,再执行级联。
*/
private async applyForeignKeyUpdateRules(
tableName: string,
oldPk: string,
newPk: string,
walRecords: Omit<import('./types').WALRecord, 'lsn' | 'checksum'>[],
visited: Set<string>,
): Promise<void> {
const visitKey = `${tableName}:${oldPk}`;
if (visited.has(visitKey)) return;
visited.add(visitKey);
// 阶段 1: RESTRICT 检查
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName) continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate) continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName) continue;
if (colDef.onUpdate !== 'RESTRICT') continue;
const refRows = await this.getAllRows(refTableName);
if (refRows.some((r) => String(r[colName]) === oldPk)) {
throw new DatabaseError(
`Cannot update "${tableName}" key "${oldPk}": foreign key "${colName}" in "${refTableName}" has dependent rows`,
'FOREIGN_KEY_VIOLATION',
);
}
}
}
// 阶段 2: CASCADE / SET NULL
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName) continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate) continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName) continue;
if (colDef.onUpdate !== 'CASCADE' && colDef.onUpdate !== 'SET NULL') continue;
const refRows = await this.getAllRows(refTableName);
for (const refRow of refRows) {
if (String(refRow[colName]) !== oldPk) continue;
const refPkCol = this.tablePKs.get(refTableName)!;
const refPk = String(refRow[refPkCol]);
const updatedRef = { ...refRow, [colName]: colDef.onUpdate === 'CASCADE' ? newPk : null };
const refKey = `${refTableName}:${refPk}`;
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(refKey, updatedRef);
this.mvcc.writeVersion(refTableName, refPk, updatedRef, this.currentTxnId);
} else {
this.lsm.put(refKey, updatedRef);
}
this.updateSecondaryIndexes(refTableName, refPk, updatedRef, refRow);
walRecords.push({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName: refTableName,
key: refPk,
data: updatedRef,
});
}
}
}
}
async delete(tableName: string, query: QueryPlan): Promise<number> {
@@ -707,6 +950,7 @@ export class AriaEngine implements IStorageEngine {
column: import('../../constants').ColumnDef & { name: string },
): Promise<void> {
this.ensureOpen();
this.ensureNoDDLInTransaction('ALTER TABLE');
this.ensureTable(tableName);
const schema = this.schemas.get(tableName)!;
@@ -723,11 +967,21 @@ export class AriaEngine implements IStorageEngine {
if (!schema.columns[column.name]) {
throw new DatabaseError(`Column "${column.name}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
}
// v0.4.2-fix: 被删列是索引列 → 先清理索引 LSM(残留会导致后续同名列索引脏数据)
if (schema.columns[column.name].index || schema.columns[column.name].unique) {
const idxKey = `${tableName}:idx:${column.name}`;
const idxLsm = this.secondaryIndexes.get(idxKey);
if (idxLsm) {
try {
await idxLsm.clear();
} catch { /* 清理失败不阻塞 */ }
this.secondaryIndexes.delete(idxKey);
}
}
delete schema.columns[column.name];
await this.persistSchemas();
// 重写主 LSM:移除所有行的该列键(find 副本无法就地删除,必须重写存储)
const pkCol = this.tablePKs.get(tableName)!;
const prefix = `${tableName}:`;
const endKey = `${prefix}\uffff`;
await this.lsm.prefetchRange(prefix, endKey);
@@ -757,30 +1011,30 @@ export class AriaEngine implements IStorageEngine {
async createIndex(tableName: string, column: string, unique?: boolean): Promise<void> {
this.ensureOpen();
this.ensureNoDDLInTransaction('CREATE INDEX');
this.ensureTable(tableName);
const schema = this.schemas.get(tableName)!;
const colDef = schema.columns[column];
if (!colDef) throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
if (colDef.index || colDef.unique) return; // 已存在
const idxKey = `${tableName}:idx:${column}`;
// v0.4.2-fix: 以索引 LSM 是否已建为准(schema 标记可能因重启恢复而存在,
// 但索引 LSM 未恢复 → 此前静默 return 导致索引永久缺失)
if (this.secondaryIndexes.has(idxKey)) return;
colDef.index = true;
if (unique) colDef.unique = true;
const idxKey = `${tableName}:idx:${column}`;
if (!this.secondaryIndexes.has(idxKey)) {
const idxLsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore: this.createSSTableStore(`idx_${tableName}_${column}`),
});
await idxLsm.init();
this.secondaryIndexes.set(idxKey, idxLsm);
}
const idxLsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore: this.createSSTableStore(`idx_${tableName}_${column}`),
});
await idxLsm.init();
this.secondaryIndexes.set(idxKey, idxLsm);
// 从主 LSM 重建索引数据
const idxLsm = this.secondaryIndexes.get(idxKey)!;
const pkCol = this.tablePKs.get(tableName)!;
const rows = await this.getAllRows(tableName);
for (const row of rows) {
@@ -795,6 +1049,7 @@ export class AriaEngine implements IStorageEngine {
async dropIndex(tableName: string, column: string, _indexName?: string): Promise<void> {
this.ensureOpen();
this.ensureNoDDLInTransaction('DROP INDEX');
this.ensureTable(tableName);
const schema = this.schemas.get(tableName)!;
const colDef = schema.columns[column];
@@ -1185,6 +1440,8 @@ export class AriaEngine implements IStorageEngine {
*/
private async applyDropTableRecovery(tableName: string): Promise<void> {
if (!tableName) return;
// v0.4.2-fix: 清理该表二级索引(崩溃恢复路径同样不留孤儿索引)
await this.cleanupTableIndexes(tableName);
this.schemas.delete(tableName);
this.tablePKs.delete(tableName);
@@ -1442,7 +1699,13 @@ export class AriaEngine implements IStorageEngine {
async reindexTable(tableName: string): Promise<number> {
this.ensureOpen();
this.ensureTable(tableName);
const schema = this.schemas.get(tableName)!;
return this.reindexTableInternal(tableName);
}
/** v0.4.2-fix: 重建索引内部实现(不校验 opened,供 open 恢复流程调用) */
private async reindexTableInternal(tableName: string): Promise<number> {
const schema = this.schemas.get(tableName);
if (!schema) return 0;
let rebuiltCount = 0;
for (const [colName, colDef] of Object.entries(schema.columns)) {
@@ -1482,7 +1745,7 @@ export class AriaEngine implements IStorageEngine {
}
}
// GC MVCC 版本(保留最新 10 个)
const beforeGC = this.mvcc.getActiveTxnCount?.() ?? 0;
const beforeGC = this.mvcc.getGlobalLSN();
this.mvcc.gc(10);
return { compactedLevels: 6, gcVersions: beforeGC };
}
@@ -1525,6 +1788,16 @@ export class AriaEngine implements IStorageEngine {
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
}
/** v0.4.2-fix: Aria 事务中 DDL 显式拒绝(结构变更无法通过行快照回滚) */
private ensureNoDDLInTransaction(op: string): void {
if (this.currentTxnId) {
throw new DatabaseError(
`${op} is not supported inside a transaction (AriaEngine DDL is not transactional)`,
'NOT_SUPPORTED',
);
}
}
private ensureTable(tableName: string): void {
if (!this.schemas.has(tableName)) {
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');