feat: v0.2.0 AriaEngine 自研存储引擎
CI / test (20.x) (push) Canceled after 0s
CI / test (22.x) (push) Canceled after 0s
CI / test (24.x) (push) Canceled after 0s
CI / test (18.x) (push) Canceled after 1h26m17s

- 新增 AriaEngine: LSM-Tree 页面式存储引擎,19 个模块,~3500 行 TS
  - page/: Slotted Page 格式 (header/slot/tuple/format) + CRC32
  - buffer/: Buffer Pool (LRU 缓存 + 驱逐策略)
  - index/: LSM-Tree (MemTable 红黑树 + SSTable + Bloom Filter + Merge Iterator)
  - wal/: WAL 日志 (二进制格式) + Checkpoint 管理
  - transaction/: MVCC 版本链 + 快照隔离
  - store/: IndexedDB / Memory 双后端抽象
  - compression/: LZ4 页面压缩

- 完整持久化: Schema 自动保存、SSTable 元数据管理、WAL 恢复
- 事务感知 CRUD: insert/update/delete 在事务中缓冲到 snapshot
- mode: 'aria' 激活自研引擎

- 新增 7 个测试文件,测试数 318 → 524,套件 20 → 27
  - aria-page.test.ts (32 tests): Page 格式单元测试
  - aria-index.test.ts (26 tests): Bloom Filter + MemTable
  - aria-sstable.test.ts (9 tests): SSTable Builder + Reader
  - aria-buffer.test.ts (25 tests): LRU + Eviction + Buffer Pool
  - aria-wal-mvcc.test.ts (22 tests): WAL 编解码 + MVCC 事务
  - aria-compress.test.ts (11 tests): LZ4 + Merge Iterator
  - aria.test.ts (80 tests): AriaEngine 集成 + 边界测试

- Bug 修复: LRUList size 跟踪、WAL 缓冲区越界、ColumnEncoding 导入
- 全面更新 README.md + site/ 站点文件 (index/docs/demo)
This commit is contained in:
2026-07-27 16:40:29 +08:00
parent c00738aea0
commit f84673e519
47 changed files with 14553 additions and 108 deletions
+704
View File
@@ -0,0 +1,704 @@
/**
* AriaEngine — 自研页面式存储引擎主类
* @module engine/aria/index
*
* 实现 IStorageEngine 接口。
*
* v0.2.1: 完整持久化
* - Schema 存入 __aria_schemas
* - SSTable 元数据存入 __aria_lsm_meta
* - WAL 恢复包含行数据
* - 启动时自动加载 Schema + SSTable
*/
import type { IStorageEngine } from '../interface';
import type { QueryPlan, TableSchema, ColumnDef } from '../../constants';
import { DatabaseError } from '../../constants';
import { matchWhere, applyOrderBy, projectColumns } from '../../query/where-matcher';
import type { AriaEngineConfig, SSTableMeta } from './types';
import { DEFAULT_ARIA_CONFIG } from './types';
import { LSM } from './index/lsm';
import type { SSTableStore } from './index/lsm';
import { WAL } from './wal/log';
import { WALRecordType, type WALRecord } from './types';
import { CheckpointManager } from './wal/checkpoint';
import { IndexedDBBackend, MemoryBackend, type IStorageBackend } from './store/backend';
// ---------------------------------------------------------------------------
// AriaEngine
// ---------------------------------------------------------------------------
export class AriaEngine implements IStorageEngine {
readonly name = 'aria';
private config!: Required<AriaEngineConfig>;
private lsm!: LSM;
private wal!: WAL;
private checkpointManager!: CheckpointManager;
private backend!: IStorageBackend;
private opened = false;
private dbName = '';
// 表结构
private schemas: Map<string, TableSchema> = new Map();
private tablePKs: Map<string, string> = new Map();
private opCounter = 0;
// 事务
private currentTxnId: number | null = null;
private txnSnapshot: Map<string, Record<string, unknown>> | null = null;
constructor(config: AriaEngineConfig = {}) {
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
}
// =======================================================================
// 生命周期
// =======================================================================
async open(dbName: string, _version: number): Promise<void> {
if (this.opened) return;
this.dbName = dbName;
// 1. 存储后端
if (this.config.storageBackend === 'indexeddb') {
this.backend = new IndexedDBBackend();
} else {
this.backend = new MemoryBackend();
}
await this.backend.open(dbName);
// 2. 构建 SSTableStore
const sstableStore = this.createSSTableStore();
// 3. 初始化 LSM
this.lsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
sstableStore,
});
// 4. 初始化 WAL
this.wal = new WAL(
{
append: async (data) => {
// Store each record as a separate numbered key
const idx = await this.getWALCount();
const slice = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
const copy = slice.buffer.slice(slice.byteOffset, slice.byteOffset + slice.byteLength) as ArrayBuffer;
await this.backend.write(`__wal_${idx}`, copy);
await this.setWALCount(idx + 1);
},
readAll: async () => {
const count = await this.getWALCount();
if (count === 0) return new Uint8Array(0);
// Read all records and concatenate
const chunks: Uint8Array[] = [];
for (let i = 0; i < count; i++) {
const d = await this.backend.read(`__wal_${i}`);
if (d) chunks.push(new Uint8Array(d));
}
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
const combined = new Uint8Array(total);
let off = 0;
for (const c of chunks) { combined.set(c, off); off += c.byteLength; }
return combined;
},
truncate: async () => {
const count = await this.getWALCount();
for (let i = 0; i < count; i++) {
await this.backend.delete(`__wal_${i}`);
}
await this.setWALCount(0);
},
exists: async () => {
const count = await this.getWALCount();
return count > 0;
},
},
this.config.walEnabled,
this.config.walSyncMode,
);
// 5. 恢复 Schema
await this.loadSchemas();
// 6. 初始化 LSM(加载 SSTable 元数据)
await this.lsm.init();
// 7. WAL 恢复(恢复未刷盘的数据)
await this.wal.recover((record) => this.applyWALRecord(record));
// 8. Checkpoint ManagerBufferPool 暂简化,使用 flush 替代)
this.checkpointManager = new CheckpointManager(
this.lsm,
this.wal,
{ flushAll: async () => { await this.lsm.flush(); } } as any,
this.config.checkpointInterval,
);
this.opened = true;
}
async close(): Promise<void> {
if (!this.opened) return;
await this.persistSchemas();
await this.lsm.flush();
await this.wal.flush();
await this.backend.close();
this.schemas.clear();
this.opened = false;
}
isOpen(): boolean { return this.opened; }
// =======================================================================
// 表管理
// =======================================================================
async createTable(schema: TableSchema): Promise<void> {
this.ensureOpen();
if (this.schemas.has(schema.name)) {
throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS');
}
this.schemas.set(schema.name, schema);
this.tablePKs.set(schema.name, this.getPK(schema));
await this.persistSchemas();
this.wal.append({
type: WALRecordType.CREATE_TABLE,
txnId: 0,
tableName: schema.name,
key: '',
data: { schema: JSON.stringify(schema) } as unknown as Record<string, unknown>,
});
}
async dropTable(tableName: string): Promise<void> {
this.ensureOpen();
this.ensureTable(tableName);
// 删除表中所有行
const rows = this.getAllRows(tableName);
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName)!;
this.lsm.delete(`${tableName}:${row[pkCol]}`);
}
this.schemas.delete(tableName);
this.tablePKs.delete(tableName);
await this.persistSchemas();
this.wal.append({
type: WALRecordType.DROP_TABLE,
txnId: 0,
tableName,
key: '',
});
}
async hasTable(tableName: string): Promise<boolean> {
return this.schemas.has(tableName);
}
async getTableNames(): Promise<string[]> {
return Array.from(this.schemas.keys());
}
async getTableSchema(tableName: string): Promise<TableSchema | null> {
return this.schemas.get(tableName) ?? null;
}
// =======================================================================
// CRUD
// =======================================================================
async insert(tableName: string, rows: Record<string, unknown>[]): Promise<string[]> {
this.ensureOpen();
this.ensureTable(tableName);
const schema = this.schemas.get(tableName)!;
const pkCol = this.tablePKs.get(tableName)!;
const pks: string[] = [];
for (const row of rows) {
const validated = this.validateRow(schema, row);
const pkValue = String(validated[pkCol]);
const key = `${tableName}:${pkValue}`;
// Check duplicate in LSM + transaction snapshot
const existing = this.currentTxnId
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
: this.lsm.get(key);
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
throw new DatabaseError(
`Duplicate primary key "${pkValue}" in table "${tableName}"`,
'DUPLICATE_KEY',
);
}
if (this.currentTxnId && this.txnSnapshot) {
// Within transaction: buffer to snapshot
this.txnSnapshot.set(key, validated);
} else {
// Direct write to LSM
this.lsm.put(key, validated);
}
pks.push(pkValue);
this.wal.append({
type: WALRecordType.INSERT,
txnId: this.currentTxnId ?? 0,
tableName,
key: pkValue,
data: validated,
});
}
this.opCounter += rows.length;
await this.checkpointManager.tick();
return pks;
}
async find(tableName: string, query: QueryPlan): Promise<Record<string, unknown>[]> {
this.ensureOpen();
this.ensureTable(tableName);
let rows: Record<string, unknown>[];
// Try index lookup
const fastPath = this.tryIndexLookup(tableName, query);
if (fastPath !== null) {
rows = fastPath;
} else {
rows = this.getAllRows(tableName);
}
// Merge transaction snapshot writes (uncommitted data visible within txn)
if (this.currentTxnId && this.txnSnapshot) {
const pkCol = this.tablePKs.get(tableName)!;
const prefix = `${tableName}:`;
for (const [key, value] of this.txnSnapshot) {
if (!key.startsWith(prefix)) continue;
const pk = key.slice(prefix.length);
const del = (value as unknown as Record<string, unknown>).__txn_deleted;
const idx = rows.findIndex((r) => r[pkCol] === pk);
if (del) {
if (idx >= 0) rows.splice(idx, 1);
} else {
const row = { ...value, [pkCol]: pk };
if (idx >= 0) rows[idx] = row;
else rows.push(row);
}
}
}
// WHERE filter
if (query.where && Object.keys(query.where).length > 0) {
rows = rows.filter((row) => matchWhere(row, query.where!));
}
// ORDER
if (query.orderBy && query.orderBy.length > 0) {
rows = applyOrderBy(rows, query.orderBy);
}
// LIMIT/OFFSET
const offset = query.offset ?? 0;
const limit = query.limit ?? rows.length;
rows = rows.slice(offset, offset + limit);
// Column projection
if (query.columns && query.columns.length > 0 && query.columns[0] !== '*') {
rows = rows.map((row) => projectColumns(row, query.columns!));
}
return rows;
}
async update(
tableName: string,
query: QueryPlan,
updates: Record<string, unknown>,
): Promise<number> {
this.ensureOpen();
this.ensureTable(tableName);
const schema = this.schemas.get(tableName)!;
const rows = this.getAllRows(tableName);
let count = 0;
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName)!;
const key = `${tableName}:${row[pkCol]}`;
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
const updated = { ...row, ...updates };
this.validateRow(schema, updated);
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(key, updated);
} else {
this.lsm.put(key, updated);
}
count++;
this.wal.append({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName,
key: String(row[pkCol]),
data: updated,
});
}
}
this.opCounter += count;
await this.checkpointManager.tick();
return count;
}
async delete(tableName: string, query: QueryPlan): Promise<number> {
this.ensureOpen();
this.ensureTable(tableName);
const rows = this.getAllRows(tableName);
let count = 0;
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName)!;
const key = `${tableName}:${row[pkCol]}`;
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
if (this.currentTxnId && this.txnSnapshot) {
// Buffer delete in snapshot
this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record<string, unknown>);
} else {
this.lsm.delete(key);
}
count++;
this.wal.append({
type: WALRecordType.DELETE,
txnId: this.currentTxnId ?? 0,
tableName,
key: String(row[pkCol]),
});
}
}
this.opCounter += count;
await this.checkpointManager.tick();
return count;
}
async count(tableName: string, query?: QueryPlan): Promise<number> {
this.ensureOpen();
const rows = this.getAllRows(tableName);
if (!query?.where || Object.keys(query.where).length === 0) return rows.length;
return rows.filter((row) => matchWhere(row, query.where!)).length;
}
async clear(tableName: string): Promise<void> {
this.ensureOpen();
this.ensureTable(tableName);
const rows = this.getAllRows(tableName);
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName)!;
this.lsm.delete(`${tableName}:${row[pkCol]}`);
}
}
// =======================================================================
// 事务
// =======================================================================
async beginTransaction(): Promise<void> {
if (this.currentTxnId) throw new DatabaseError('Transaction already in progress', 'TX_ACTIVE');
this.currentTxnId = Date.now();
this.txnSnapshot = new Map();
this.wal.append({
type: WALRecordType.BEGIN,
txnId: this.currentTxnId,
tableName: '',
key: '',
});
}
async commitTransaction(): Promise<void> {
if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE');
if (this.txnSnapshot) {
for (const [key, value] of this.txnSnapshot) {
if ((value as unknown as Record<string, unknown>).__txn_deleted) {
this.lsm.delete(key);
} else {
this.lsm.put(key, value);
}
}
}
this.wal.append({
type: WALRecordType.COMMIT,
txnId: this.currentTxnId,
tableName: '',
key: '',
});
this.currentTxnId = null;
this.txnSnapshot = null;
await this.wal.flush();
}
async rollbackTransaction(): Promise<void> {
if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE');
this.txnSnapshot = null;
this.wal.append({
type: WALRecordType.ROLLBACK,
txnId: this.currentTxnId,
tableName: '',
key: '',
});
this.currentTxnId = null;
}
// =======================================================================
// 内部
// =======================================================================
private getAllRows(tableName: string): Record<string, unknown>[] {
const pkCol = this.tablePKs.get(tableName)!;
const prefix = `${tableName}:`;
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
return entries.map(([key, value]) => {
const row = { ...value };
row[pkCol] = key.slice(prefix.length);
return row;
});
}
private tryIndexLookup(
tableName: string,
query: QueryPlan,
): Record<string, unknown>[] | null {
if (!query.where) return null;
const pkCol = this.tablePKs.get(tableName)!;
for (const [col, condition] of Object.entries(query.where)) {
if (col !== pkCol) continue;
// 等值条件
if (typeof condition !== 'object' || condition === null) {
const key = `${tableName}:${condition}`;
const value = this.lsm.get(key);
return value ? [{ ...value, [pkCol]: condition }] : [];
}
const cond = condition as Record<string, unknown>;
if ('$eq' in cond) {
const key = `${tableName}:${cond.$eq}`;
const value = this.lsm.get(key);
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
}
}
return null;
}
private getPK(schema: TableSchema): string {
for (const [name, col] of Object.entries(schema.columns)) {
if (col.primaryKey) return name;
}
return Object.keys(schema.columns)[0];
}
private validateRow(schema: TableSchema, row: Record<string, unknown>): Record<string, unknown> {
const validated: Record<string, unknown> = {};
for (const [colName, colDef] of Object.entries(schema.columns)) {
let value = row[colName];
if (value === undefined && colDef.default !== undefined) value = colDef.default;
if (colDef.required && (value === undefined || value === null)) {
throw new DatabaseError(`Column "${colName}" is required in table "${schema.name}"`, 'VALIDATION_ERROR');
}
if (value !== undefined && value !== null) {
this.checkType(colName, colDef.type, value);
}
if (value !== undefined) validated[colName] = value;
}
return validated;
}
private checkType(colName: string, type: string, value: unknown): void {
const jsType = typeof value;
switch (type) {
case 'string': if (jsType !== 'string') throw new DatabaseError(`Column "${colName}" expects string, got ${jsType}`, 'TYPE_ERROR'); break;
case 'number': if (jsType !== 'number') throw new DatabaseError(`Column "${colName}" expects number, got ${jsType}`, 'TYPE_ERROR'); break;
case 'boolean': if (jsType !== 'boolean') throw new DatabaseError(`Column "${colName}" expects boolean, got ${jsType}`, 'TYPE_ERROR'); break;
case 'date': if (jsType !== 'string' || isNaN(Date.parse(value as string))) throw new DatabaseError(`Column "${colName}" expects valid date`, 'TYPE_ERROR'); break;
case 'json': if (jsType !== 'object') throw new DatabaseError(`Column "${colName}" expects object/array, got ${jsType}`, 'TYPE_ERROR'); break;
}
}
// =======================================================================
// Schema 持久化
// =======================================================================
private async persistSchemas(): Promise<void> {
const data: Record<string, Record<string, ColumnDef>> = {};
for (const [name, schema] of this.schemas) {
data[name] = schema.columns;
}
const json = JSON.stringify(data);
const buf = new TextEncoder().encode(json).buffer;
await this.backend.write('__aria_schemas', buf);
}
private async loadSchemas(): Promise<void> {
const raw = await this.backend.read('__aria_schemas');
if (!raw) return;
try {
const json = new TextDecoder().decode(raw);
const data = JSON.parse(json) as Record<string, Record<string, ColumnDef>>;
for (const [tableName, columns] of Object.entries(data)) {
const schema: TableSchema = { name: tableName, columns };
this.schemas.set(tableName, schema);
this.tablePKs.set(tableName, this.getPK(schema));
}
} catch {
// 忽略损坏的 schema 数据
}
}
// =======================================================================
// SSTableStore 构建
// =======================================================================
private createSSTableStore(): SSTableStore {
const META_KEY = '__aria_lsm_meta';
return {
save: async (id, data) => {
const buf = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
await this.backend.write(`sst_${id}`, buf);
},
load: async (id) => {
const buf = await this.backend.read(`sst_${id}`);
return buf ? new Uint8Array(buf) : null;
},
delete: async (id) => {
await this.backend.delete(`sst_${id}`);
},
allocateId: async () => Date.now(),
listMeta: async () => {
const raw = await this.backend.read(META_KEY);
if (!raw) return [];
try {
return JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[];
} catch {
return [];
}
},
saveMeta: async (meta) => {
const existing = await this.backend.read(META_KEY);
const list: SSTableMeta[] = existing
? JSON.parse(new TextDecoder().decode(existing))
: [];
// 更新或添加
const idx = list.findIndex((m) => m.id === meta.id);
if (idx >= 0) list[idx] = meta;
else list.push(meta);
const json = JSON.stringify(list);
const buf = new TextEncoder().encode(json).buffer;
await this.backend.write(META_KEY, buf);
},
deleteMeta: async (id) => {
const existing = await this.backend.read(META_KEY);
if (!existing) return;
const list: SSTableMeta[] = JSON.parse(new TextDecoder().decode(existing));
const filtered = list.filter((m) => m.id !== id);
const json = JSON.stringify(filtered);
const buf = new TextEncoder().encode(json).buffer;
await this.backend.write(META_KEY, buf);
},
};
}
// =======================================================================
// WAL 恢复
// =======================================================================
private applyWALRecord(record: WALRecord): void {
switch (record.type) {
case WALRecordType.INSERT:
case WALRecordType.UPDATE:
if (record.data) {
this.lsm.put(`${record.tableName}:${record.key}`, record.data);
}
break;
case WALRecordType.DELETE:
this.lsm.delete(`${record.tableName}:${record.key}`);
break;
case WALRecordType.CREATE_TABLE:
if (record.data?.schema) {
try {
const s = JSON.parse(record.data.schema as string) as TableSchema;
if (!this.schemas.has(s.name)) {
this.schemas.set(s.name, s);
this.tablePKs.set(s.name, this.getPK(s));
}
} catch { /* skip */ }
}
break;
case WALRecordType.COMMIT:
case WALRecordType.ROLLBACK:
case WALRecordType.BEGIN:
break;
}
}
// =======================================================================
// 辅助
// =======================================================================
private ensureOpen(): void {
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
}
private ensureTable(tableName: string): void {
if (!this.schemas.has(tableName)) {
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
}
}
/** Get the number of WAL records stored */
private async getWALCount(): Promise<number> {
const raw = await this.backend.read('__wal_count');
if (!raw) return 0;
try {
const dec = new TextDecoder();
return parseInt(dec.decode(raw), 10) || 0;
} catch {
return 0;
}
}
/** Set the number of WAL records stored */
private async setWALCount(count: number): Promise<void> {
const enc = new TextEncoder();
const buf = enc.encode(String(count)).buffer;
await this.backend.write('__wal_count', buf);
}
}