Files
MetonaSqlark/dist/metona-sqlark.esm.js
T
thzxx 05e6823bf1
CI / test (22.x) (push) Successful in 24m26s
CI / e2e (push) Successful in 10m0s
CI / test (18.x) (push) Successful in 27m14s
CI / test (20.x) (push) Failing after 1h19m9s
CI / test (24.x) (push) Successful in 37m49s
fix: v0.7.2 语句级原子性 + 事务 DDL 拒绝 + 约束/绑定硬化 — 6 项修复 + 43 回归 + CI 重型套件串行
- UPDATE 语句级部分提交(P1,四引擎):两阶段全量预检后执行,批内唯一互查,
  任何一行失败整句不执行(aria 场景 WAL 与内存不再错位)
- 事务内 ALTER/CREATE INDEX/DROP INDEX 残留(P1):Memory/KVStore 显式拒绝
  (对齐 Aria),createTable/dropTable 保持可回滚
- SET NULL 级联绕过 required 约束(P1):预检阶段整体拒绝 FOREIGN_KEY_VIOLATION
- bindParameters 注释误判(P2):行注释/块注释中的 ? 与引号不再参与绑定
- 未闭合字符串静默接受 → lexer 抛 PARSE_ERROR;未知 where 操作符抛 QUERY_ERROR
- UPDATE undefined 覆盖列值 → 语义化为不更新(null 仍置空)
- Hybrid 写穿透非原子(P1):磁盘失败自动重载内存对齐磁盘再抛原错误
- CI:Run tests 拆常规并行 + 重型串行(runInBand),重型测试超时余量提升,
  性能护栏 kv 120→240s / opfs 150→300s(仍拦截悬崖回归)
- 测试 1155 → 1198(74 套件),覆盖率 89.82% 保持
2026-08-13 15:31:16 +08:00

12828 lines
504 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* metona-sqlark Constants — 类型定义 / 默认配置 / 枚举
* @module constants
*/
/** 所有字段类型 */
const FIELD_TYPES = ['string', 'number', 'boolean', 'date', 'json'];
/** 数据库默认配置 */
const DB_DEFAULTS = Object.freeze({
name: 'metona-sqlark',
mode: 'hybrid',
diskEngine: 'opfs',
version: 1,
maxRowsPerQuery: 0, // 0 = 不限制
debug: false,
multiTabSync: false,
aria: undefined,
});
// ---------------------------------------------------------------------------
// 错误类型
// ---------------------------------------------------------------------------
/** 数据库错误 */
class DatabaseError extends Error {
constructor(message, code, details) {
super(message);
this.code = code;
this.details = details;
this.name = 'DatabaseError';
}
}
// ---------------------------------------------------------------------------
// 版本
// ---------------------------------------------------------------------------
const VERSION = '0.7.2';
/**
* metona-sqlark Shared WHERE Matcher — 统一的条件匹配逻辑
* @module query/where-matcher
*
* MemoryEngine / IndexedDBEngine / QueryExecutor 共享此模块,
* 消除 220+ 行重复代码,统一 $and/$or/$not/$col 行为。
*/
// ---------------------------------------------------------------------------
// LIKE 正则缓存
// ---------------------------------------------------------------------------
const likeCache = new Map();
function compileLikeRegex(pattern) {
const cached = likeCache.get(pattern);
if (cached)
return cached;
const escaped = pattern
.replace(/[.+^${}()|[\]\\]/g, '\\$&')
.replace(/%/g, '.*')
.replace(/_/g, '.');
const regex = new RegExp(`^${escaped}$`, 'i');
likeCache.set(pattern, regex);
return regex;
}
// ---------------------------------------------------------------------------
// WHERE 匹配(顶层入口)
// ---------------------------------------------------------------------------
/**
* 匹配完整 WHERE 条件
* @param row 当前数据行
* @param where WHERE 条件对象
* @param options.$col 是否启用 $col 列引用解析
*/
function matchWhere(row, where, options = {}) {
for (const [field, condition] of Object.entries(where)) {
// 顶层 $caseResultv0.3.2):由 Executor 对 CASE WHEN 表达式逐行求值后产生
if (field === '$caseResult') {
if (condition !== true)
return false;
continue;
}
// 顶层 $existsv0.3.0):由 Executor.resolveSubqueries 解析为 boolean
if (field === '$exists') {
if (condition !== true)
return false;
continue;
}
// 顶层 $and
if (field === '$and') {
const subs = condition;
if (!subs.every((sub) => matchWhere(row, sub, options)))
return false;
continue;
}
// 顶层 $or
if (field === '$or') {
const subs = condition;
if (!subs.some((sub) => matchWhere(row, sub, options)))
return false;
continue;
}
// 顶层 $notv0.3.2 修复:NOT (expr) 生成的 { $not: inner }
if (field === '$not') {
if (matchWhere(row, condition, options))
return false;
continue;
}
if (!matchField(row[field], condition, row, options))
return false;
}
return true;
}
// ---------------------------------------------------------------------------
// 字段匹配
// ---------------------------------------------------------------------------
function matchField(value, condition, row, options) {
// 嵌套 $and
if (typeof condition === 'object' && condition !== null && '$and' in condition) {
const subs = condition.$and;
return subs.every((sub) => matchWhere(row, sub, options));
}
// 嵌套 $or
if (typeof condition === 'object' && condition !== null && '$or' in condition) {
const subs = condition.$or;
return subs.some((sub) => matchWhere(row, sub, options));
}
// $not
if (typeof condition === 'object' && condition !== null && '$not' in condition) {
return !matchField(value, condition.$not, row, options);
}
// 简单值 => $eq
if (typeof condition !== 'object' || condition === null || Array.isArray(condition)) {
return value === condition;
}
const ops = condition;
// $col 简写: { $col: name } === { $eq: { $col: name } }(仅 JOIN ON 场景)
if (options.$col && '$col' in ops && Object.keys(ops).length === 1) {
return value === row[ops.$col];
}
// 遍历操作符
for (const [op, operand] of Object.entries(ops)) {
let actualOperand = operand;
// $col 列引用解析
if (options.$col && typeof operand === 'object' && operand !== null && '$col' in operand) {
actualOperand = row[operand.$col];
}
if (!matchOperator(value, op, actualOperand))
return false;
}
return true;
}
// ---------------------------------------------------------------------------
// 操作符匹配
// ---------------------------------------------------------------------------
function matchOperator(value, op, operand) {
switch (op) {
case '$eq': return value === operand;
case '$ne': return value !== operand;
case '$gt': return value > operand;
case '$gte': return value >= operand;
case '$lt': return value < operand;
case '$lte': return value <= operand;
case '$in': return Array.isArray(operand) && operand.includes(value);
case '$nin': return Array.isArray(operand) && !operand.includes(value);
case '$like': return compileLikeRegex(String(operand)).test(String(value));
// v0.7.2: 未知操作符显式报错 —— 此前静默返回 true(所有行匹配),
// 拼错操作符(如 $betwen)时过滤形同虚设且无任何提示
default:
throw new DatabaseError(`Unknown where operator "${op}"`, 'QUERY_ERROR');
}
}
// ---------------------------------------------------------------------------
// 排序
// ---------------------------------------------------------------------------
function applyOrderBy(rows, orderBy) {
return [...rows].sort((a, b) => {
for (const { column, direction, nulls } of orderBy) {
const aNull = a[column] === null || a[column] === undefined;
const bNull = b[column] === null || b[column] === undefined;
// v0.4.0: NULLS FIRST/LAST 时 NULL 位置固定,不受升降序反转
if (nulls && (aNull || bNull)) {
if (aNull && bNull)
continue;
const cmp = nulls === 'first' ? (aNull ? -1 : 1) : (aNull ? 1 : -1);
return cmp;
}
const cmp = compare(a[column], b[column]);
if (cmp !== 0)
return direction === 'desc' ? -cmp : cmp;
}
return 0;
});
}
function compare(a, b) {
if (a === b)
return 0;
if (a === null || a === undefined)
return 1;
if (b === null || b === undefined)
return -1;
if (typeof a === 'string' && typeof b === 'string')
return a.localeCompare(b);
if (typeof a === 'number' && typeof b === 'number')
return a - b;
return String(a).localeCompare(String(b));
}
// ---------------------------------------------------------------------------
// 列投影
// ---------------------------------------------------------------------------
function projectColumns(row, columns) {
const projected = {};
for (const col of columns) {
if (col in row) {
projected[col] = row[col];
}
else {
for (const key of Object.keys(row)) {
if (key.endsWith(`.${col}`) || key === col) {
projected[col] = row[key];
break;
}
}
}
}
return projected;
}
/**
* metona-sqlark Schema — 表结构定义与校验
* @module table/schema
*/
// ---------------------------------------------------------------------------
// Schema 工具
// ---------------------------------------------------------------------------
/** 从列定义创建 TableSchema */
function createSchema(name, columns) {
validateColumns(columns);
return { name, columns };
}
/** 校验列定义 */
function validateColumns(columns) {
const colNames = Object.keys(columns);
if (colNames.length === 0) {
throw new DatabaseError('Table must have at least one column', 'SCHEMA_ERROR');
}
let primaryKeyCount = 0;
for (const [colName, colDef] of Object.entries(columns)) {
// v0.7.1: '__proto__' 作为列名会触发对象原型 setter(列静默丢失);
// 显式拒绝避免原型污染类攻击面
if (colName === '__proto__') {
throw new DatabaseError('Column name "__proto__" is not allowed', 'SCHEMA_ERROR');
}
// 类型校验
if (!FIELD_TYPES.includes(colDef.type)) {
throw new DatabaseError(`Invalid type "${colDef.type}" for column "${colName}". Valid types: ${FIELD_TYPES.join(', ')}`, 'SCHEMA_ERROR');
}
// 主键计数
if (colDef.primaryKey) {
primaryKeyCount++;
}
}
// 至少需要一个主键
if (primaryKeyCount === 0) {
throw new DatabaseError('Table must have at least one primary key column', 'SCHEMA_ERROR');
}
// v0.7.0: 复合主键(多列 primaryKey)当前不支持 —— 所有引擎的存储布局与
// 外键引用均为单主键假设(此前静默取第一个主键,其余标记被忽略 → 语义陷阱)。
// 显式拒绝避免用户误用;复合主键列入 v0.8 路线图。
if (primaryKeyCount > 1) {
throw new DatabaseError(`Composite primary keys are not supported yet: table has ${primaryKeyCount} primary key columns. ` +
'Use a single primary key column (or a unique column combination) instead.', 'SCHEMA_ERROR');
}
}
/**
* v0.7.2: 更新载荷清洗 —— undefined 值视为"不更新该列"(保留旧值)。
* 此前 `update({ col: undefined })` 会把 undefined 写入行(覆盖旧值、列键丢失)。
* null 保留(显式置空语义)。
*/
function stripUndefinedUpdates(updates) {
const clean = {};
for (const [key, value] of Object.entries(updates)) {
if (value !== undefined)
clean[key] = value;
}
return clean;
}
/** 检查字段类型(含约束校验) */
function checkFieldType(tableName, colName, type, value, colDef) {
const jsType = typeof value;
switch (type) {
case 'string':
if (jsType !== 'string') {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" expects string, got ${jsType}`, 'TYPE_ERROR');
}
if (colDef?.maxLength !== undefined && value.length > colDef.maxLength) {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" exceeds max length ${colDef.maxLength}`, 'VALIDATION_ERROR');
}
break;
case 'number':
if (jsType !== 'number') {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" expects number, got ${jsType}`, 'TYPE_ERROR');
}
if (colDef?.min !== undefined && value < colDef.min) {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" value ${value} below minimum ${colDef.min}`, 'VALIDATION_ERROR');
}
if (colDef?.max !== undefined && value > colDef.max) {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" value ${value} above maximum ${colDef.max}`, 'VALIDATION_ERROR');
}
break;
case 'boolean':
if (jsType !== 'boolean') {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" expects boolean, got ${jsType}`, 'TYPE_ERROR');
}
break;
case 'date':
if (jsType !== 'string' || isNaN(Date.parse(value))) {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" expects valid date string, got ${typeof value}`, 'TYPE_ERROR');
}
break;
case 'json':
if (jsType !== 'object') {
throw new DatabaseError(`Column "${colName}" in table "${tableName}" expects object/array, got ${jsType}`, 'TYPE_ERROR');
}
break;
}
}
/** 将 AST 列定义转换为 ColumnDef */
function astColumnToColumnDef(astCol) {
return {
type: astCol.type,
primaryKey: astCol.primaryKey,
unique: astCol.unique,
required: astCol.required,
default: astCol.default,
index: astCol.index,
maxLength: astCol.maxLength,
min: astCol.min,
max: astCol.max,
references: astCol.references,
onDelete: astCol.onDelete,
onUpdate: astCol.onUpdate,
};
}
/**
* metona-sqlark Memory Engine — 基于 Map 的内存存储引擎
* @module engine/memory
*/
class MemoryEngine {
constructor() {
this.name = 'memory';
this.tables = new Map();
this.schemas = new Map();
this.indexes = new Map();
this.opened = false;
/** v0.4.2-fix: 库内元数据(迁移版本持久化用) */
this.metaStore = new Map();
// ---- 事务快照 ----
this.snapshot = null;
}
// ---- 生命周期 ----
async open(_dbName, _version) {
if (this.opened) {
// 幂等:已打开则忽略
return;
}
this.opened = true;
}
async close() {
this.tables.clear();
this.schemas.clear();
this.indexes.clear();
this.metaStore.clear();
this.opened = false;
}
isOpen() { return this.opened; }
// ---- v0.4.2-fix: 自愈 / 重置 / 元数据 ----
/** 内存引擎无需修复(无持久化损坏概念) */
async repair() { return; }
/** 清空全部数据与表结构 */
async clearAll() {
const names = Array.from(this.schemas.keys());
for (const name of names) {
await this.dropTable(name);
}
this.metaStore.clear();
}
async getMeta(key) {
return this.metaStore.get(key) ?? null;
}
async setMeta(key, value) {
this.metaStore.set(key, value);
}
// ---- 表管理 ----
async createTable(schema) {
if (this.schemas.has(schema.name))
throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS');
// v0.4.2-fix: 存储 schema 深拷贝 — 此前 Hybrid.reloadMemoryFromDisk 直接存入
// disk 引擎的 schema 引用,内存/磁盘引擎共享同一对象,任一引擎 ALTER 都会污染对方
const copy = { name: schema.name, columns: {} };
for (const [colName, colDef] of Object.entries(schema.columns)) {
copy.columns[colName] = { ...colDef };
}
this.schemas.set(schema.name, copy);
this.tables.set(schema.name, new Map());
const tableIndexes = new Map();
for (const [colName, colDef] of Object.entries(copy.columns)) {
if (colDef.index || colDef.unique)
tableIndexes.set(colName, new Map());
}
this.indexes.set(schema.name, tableIndexes);
}
async dropTable(tableName) {
this.ensureTable(tableName);
this.schemas.delete(tableName);
this.tables.delete(tableName);
this.indexes.delete(tableName);
}
async hasTable(tableName) { return this.schemas.has(tableName); }
async getTableNames() { return Array.from(this.schemas.keys()); }
async getTableSchema(tableName) { return this.schemas.get(tableName) ?? null; }
/**
* v0.4.2-fix: 引擎级 ALTER TABLE — 直接修改内存 schema 引用并清理行数据。
* (此前走 executor 通用路径,行为相同;统一到引擎层保证 Hybrid/IndexedDB 委托一致性)
*/
async alterTable(tableName, action, column) {
// v0.7.2: 事务内 DDL 显式拒绝(与 AriaEngine 对齐)。此前事务快照对 schema
// 是浅拷贝,alterTable 直接修改共享 columns 对象 → ROLLBACK 后结构变更残留
// (三引擎行为不一致:Aria 拒绝 / Memory、KVStore 静默残留)
if (this.snapshot) {
throw new DatabaseError(`ALTER TABLE is not supported inside a transaction (MemoryEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
if (action === 'ADD') {
if (schema.columns[column.name]) {
throw new DatabaseError(`Column "${column.name}" already exists in table "${tableName}"`, 'COLUMN_EXISTS');
}
schema.columns[column.name] = column;
return;
}
if (!schema.columns[column.name]) {
throw new DatabaseError(`Column "${column.name}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
}
delete schema.columns[column.name];
// 清理已有行中该列的值(find 返回行引用,直接删除生效)
const table = this.tables.get(tableName);
for (const row of table.values()) {
if (column.name in row)
delete row[column.name];
}
}
// ---- CRUD ----
async insert(tableName, rows) {
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const table = this.tables.get(tableName);
const pkColumn = this.getPrimaryKey(schema);
const pks = [];
for (const row of rows) {
const validatedRow = this.validateRow(schema, row);
const pkValue = String(validatedRow[pkColumn]);
if (table.has(pkValue))
throw new DatabaseError(`Duplicate primary key "${pkValue}" in table "${tableName}"`, 'DUPLICATE_KEY');
this.checkUniqueness(schema, validatedRow);
table.set(pkValue, validatedRow);
this.updateIndexes(tableName, validatedRow, pkValue);
pks.push(pkValue);
}
return pks;
}
async find(tableName, query) {
this.ensureTable(tableName);
const table = this.tables.get(tableName);
let results = this.tryIndexLookup(tableName, table, query);
if (query.where && Object.keys(query.where).length > 0) {
results = results.filter((row) => matchWhere(row, query.where));
}
if (query.orderBy && query.orderBy.length > 0) {
results = applyOrderBy(results, query.orderBy);
}
const offset = query.offset ?? 0;
const limit = query.limit ?? results.length;
results = results.slice(offset, offset + limit);
if (query.columns && query.columns.length > 0 && query.columns[0] !== '*') {
results = results.map((row) => projectColumns(row, query.columns));
}
return results;
}
/** v0.4.0: 流式查询 — 逐行回调(单次迭代,不物化结果数组) */
async findStream(tableName, query, onRow) {
this.ensureTable(tableName);
const table = this.tables.get(tableName);
const hasWhere = !!(query.where && Object.keys(query.where).length > 0);
const limit = query.limit ?? Infinity;
const offset = query.offset ?? 0;
const project = query.columns && query.columns.length > 0 && query.columns[0] !== '*'
? (row) => projectColumns(row, query.columns)
: null;
let count = 0;
let skipped = 0;
for (const row of table.values()) {
if (hasWhere && !matchWhere(row, query.where))
continue;
if (skipped < offset) {
skipped++;
continue;
}
onRow(project ? project(row) : row);
count++;
if (count >= limit)
break;
}
return count;
}
async update(tableName, query, updates) {
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const table = this.tables.get(tableName);
const pkCol = this.getPrimaryKey(schema);
// v0.7.2: undefined 值视为"不更新该列"(保留旧值),null 显式置空
const cleanUpdates = stripUndefinedUpdates(updates);
// v0.7.2: 语句级原子性 — 两阶段(先全量预检,后执行)。
// 此前逐行"校验+写入":第 N 行唯一冲突/校验失败抛错时,前 N-1 行已写入
// → 无事务下语句级部分提交(数据半更新且调用方已收到错误)。
const planned = [];
const batchUnique = new Map();
// 阶段 1:全量预检(任何一行失败 → 整条语句不执行)
for (const [pk, row] of table) {
if (query.where && Object.keys(query.where).length > 0 && !matchWhere(row, query.where))
continue;
const updated = { ...row, ...cleanUpdates };
this.validateRow(schema, updated);
this.checkUpdateUniqueness(schema, tableName, pk, updated, batchUnique);
const newPk = String(updated[pkCol]);
// v0.6.2-fix(P0): 主键变更撞已有主键 → 抛 DUPLICATE_KEY(此前静默覆盖丢数据)
if (newPk !== pk && table.has(newPk)) {
throw new DatabaseError(`Duplicate primary key "${newPk}" in table "${tableName}" (cannot update key to existing value)`, 'DUPLICATE_KEY');
}
planned.push({ pk, row, updated, newPk });
}
// 阶段 1b:主键变更 RESTRICT 预检(引用表依赖行检查,任何修改前)
for (const p of planned) {
if (p.newPk !== p.pk)
this.checkUpdateRestrict(tableName, p.pk);
}
// 阶段 2:执行(预检已通过,此阶段不再抛校验类错误)
let count = 0;
for (const { pk, row, updated, newPk } of planned) {
// v0.3.3: 先移除旧值索引条目(修复 update 后唯一约束被绕过、按新值查索引丢行)
this.removeIndexEntries(tableName, row, pk);
// v0.4.2-fix: 主键变更 — 删除旧键 + 级联更新引用表 + 新键落表
if (newPk !== pk) {
await this.applyUpdateCascade(tableName, pk, newPk);
}
table.delete(pk);
table.set(newPk, updated);
this.updateIndexes(tableName, updated, newPk);
count++;
}
return count;
}
/**
* v0.7.2: 更新唯一性预检 — 批内互查(多条行更新到同一唯一值)+ 索引查
* (排除自身旧条目)。阶段 1 中索引尚未更新,批内互查避免"两行同时改到
* 同一新值"绕过唯一约束。
*/
checkUpdateUniqueness(schema, tableName, pk, updated, batchUnique) {
const tableIndexes = this.indexes.get(tableName);
for (const [colName, colDef] of Object.entries(schema.columns)) {
if (!colDef.unique)
continue;
const value = updated[colName];
if (value === undefined || value === null)
continue;
let seen = batchUnique.get(colName);
if (!seen) {
seen = new Set();
batchUnique.set(colName, seen);
}
if (seen.has(value)) {
throw new DatabaseError(`Unique constraint violation on column "${colName}" in table "${schema.name}"`, 'UNIQUE_VIOLATION');
}
seen.add(value);
if (!tableIndexes)
continue;
const colIndex = tableIndexes.get(colName);
if (colIndex && colIndex.has(value)) {
const pks = colIndex.get(value);
// 值未变(新值 = 旧值)且索引中只有自身 → 允许
if (!(pks.size === 1 && pks.has(pk))) {
throw new DatabaseError(`Unique constraint violation on column "${colName}" in table "${schema.name}"`, 'UNIQUE_VIOLATION');
}
}
}
}
/**
* v0.7.2: ON UPDATE RESTRICT 预检 — 从 applyUpdateCascade 提取,
* 两阶段 update 在任何修改前调用(整体拒绝语义)。
*/
checkUpdateRestrict(tableName, oldPk) {
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refTableData = this.tables.get(refTableName);
if (!refTableData)
continue;
let hasDependents = false;
for (const [, refRow] of refTableData) {
if (String(refRow[colName]) !== oldPk)
continue;
hasDependents = true;
if (colDef.onUpdate === 'RESTRICT') {
throw new DatabaseError(`Cannot update "${tableName}" key "${oldPk}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
}
// v0.7.2: SET NULL 到 required 列违反约束 —— 与 RESTRICT 同样整体拒绝
// (此前级联直写 null 绕过 validateRowrequired 列被静默置空)
if (hasDependents && colDef.onUpdate === 'SET NULL' && colDef.required) {
throw new DatabaseError(`Cannot update "${tableName}" key "${oldPk}": foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`, 'FOREIGN_KEY_VIOLATION');
}
}
}
}
/**
* v0.4.2-fix: ON UPDATE 外键级联 — 被引用表主键变更时处理引用表:
* RESTRICT 抛错 / CASCADE 更新 FK 值 / SET NULL 置空。
* 分两阶段:先全量 RESTRICT 检查(任何修改前),再执行级联(防部分修改)。
*/
async applyUpdateCascade(tableName, oldPk, newPk) {
// 阶段 1: RESTRICT 检查(引用旧主键的行存在即拒绝)
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refTableData = this.tables.get(refTableName);
if (!refTableData)
continue;
for (const [, refRow] of refTableData) {
if (String(refRow[colName]) === oldPk && colDef.onUpdate === 'RESTRICT') {
throw new DatabaseError(`Cannot update "${tableName}" key "${oldPk}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
}
}
}
// 阶段 2: CASCADE / SET NULL
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refTableData = this.tables.get(refTableName);
if (!refTableData)
continue;
if (colDef.onUpdate !== 'CASCADE' && colDef.onUpdate !== 'SET NULL')
continue;
for (const [refPk, refRow] of refTableData) {
if (String(refRow[colName]) !== oldPk)
continue;
this.removeIndexEntries(refTableName, refRow, refPk);
refRow[colName] = colDef.onUpdate === 'CASCADE' ? newPk : null;
this.updateIndexes(refTableName, refRow, refPk);
}
}
}
}
async delete(tableName, query) {
this.ensureTable(tableName);
const table = this.tables.get(tableName);
const toDelete = [];
for (const [pk, row] of table) {
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
// v0.3.3: 删除行前清理其索引条目(修复删除后索引残留)
this.removeIndexEntries(tableName, row, pk);
toDelete.push(pk);
}
}
// v0.6.3-fix: 级联两阶段 —— 先对全部待删行做 RESTRICT 预检(沿 CASCADE 链递归),
// 任何一行违规则整体拒绝。此前逐行执行:第 N 行 RESTRICT 抛错时,前 N-1 行的
// 级联子行已被删除、父行未删 → 无事务下部分级联(数据不一致)
const restrictVisited = new Set();
for (const pk of toDelete) {
const row = table.get(pk);
if (row)
this.checkCascadeRestrict(tableName, pk, restrictVisited);
}
// 级联删除:检查引用此表的其他表(RESTRICT 已预检通过,此阶段不再抛错)
let cascadeCount = 0;
for (const pk of toDelete) {
const row = table.get(pk);
if (row)
cascadeCount += await this.cascadeDelete(tableName, pk, row);
}
for (const pk of toDelete)
table.delete(pk);
return toDelete.length + cascadeCount;
}
/**
* v0.6.3: RESTRICT 预检(delete 级联两阶段之一)。
* 递归沿 CASCADE 链检查引用表:RESTRICT 引用存在依赖行则抛 FOREIGN_KEY_VIOLATION。
*/
checkCascadeRestrict(tableName, pkValue, visited) {
const visitKey = `${tableName}:${pkValue}`;
if (visited.has(visitKey))
return;
visited.add(visitKey);
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onDelete)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refTableData = this.tables.get(refTableName);
if (!refTableData)
continue;
const refPks = [];
for (const [refPk, refRow] of refTableData) {
if (String(refRow[colName]) === pkValue)
refPks.push(refPk);
}
if (colDef.onDelete === 'RESTRICT' && refPks.length > 0) {
throw new DatabaseError(`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
// v0.7.2: SET NULL 到 required 列违反约束 —— 预检阶段整体拒绝
if (colDef.onDelete === 'SET NULL' && colDef.required && refPks.length > 0) {
throw new DatabaseError(`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`, 'FOREIGN_KEY_VIOLATION');
}
if (colDef.onDelete === 'CASCADE') {
for (const refPk of refPks) {
this.checkCascadeRestrict(refTableName, refPk, visited);
}
}
}
}
}
async count(tableName, query) {
this.ensureTable(tableName);
const table = this.tables.get(tableName);
if (!query?.where || Object.keys(query.where).length === 0)
return table.size;
let result = 0;
for (const row of table.values()) {
if (matchWhere(row, query.where))
result++;
}
return result;
}
async clear(tableName) {
this.ensureTable(tableName);
this.tables.get(tableName).clear();
const tableIndexes = this.indexes.get(tableName);
if (tableIndexes)
for (const colIndex of tableIndexes.values())
colIndex.clear();
}
// ---- 动态索引(v0.3.0 ----
async createIndex(tableName, column, unique) {
// v0.7.2: 事务内修改列级标志(colDef.index/unique)会写入共享列对象,
// 事务快照无法回滚 → 与 alterTable 同样显式拒绝
if (this.snapshot) {
throw new DatabaseError(`CREATE INDEX is not supported inside a transaction (MemoryEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const colDef = schema.columns[column];
if (!colDef)
throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
if (colDef.index || colDef.unique)
return; // 已存在
colDef.index = true;
if (unique)
colDef.unique = true;
const tableIndexes = this.indexes.get(tableName);
if (!tableIndexes.has(column))
tableIndexes.set(column, new Map());
const colIndex = tableIndexes.get(column);
const table = this.tables.get(tableName);
for (const [pk, row] of table) {
const value = row[column];
if (value !== undefined && value !== null) {
if (!colIndex.has(value))
colIndex.set(value, new Set());
colIndex.get(value).add(pk);
}
}
}
async dropIndex(tableName, column, _indexName) {
// v0.7.2: 同 createIndex —— 列级标志修改无法通过事务快照回滚,显式拒绝
if (this.snapshot) {
throw new DatabaseError(`DROP INDEX is not supported inside a transaction (MemoryEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const colDef = schema.columns[column];
if (!colDef)
throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
// v0.4.1: DROP 不存在的索引应报错(此前静默成功)
if (!colDef.index && !colDef.unique) {
throw new DatabaseError(`Index on column "${column}" does not exist in table "${tableName}"`, 'INDEX_NOT_FOUND');
}
colDef.index = false;
colDef.unique = false;
const tableIndexes = this.indexes.get(tableName);
if (tableIndexes)
tableIndexes.delete(column);
}
// ---- 事务 ----
async beginTransaction() {
if (this.snapshot)
throw new DatabaseError('Transaction already in progress', 'TX_ACTIVE');
this.snapshot = {
tables: this.deepCloneMapMap(this.tables),
schemas: new Map(this.schemas),
indexes: this.deepCloneIndexes(this.indexes),
};
}
async commitTransaction() {
if (!this.snapshot)
throw new DatabaseError('No active transaction', 'TX_NONE');
this.snapshot = null;
}
async rollbackTransaction() {
if (!this.snapshot)
throw new DatabaseError('No active transaction', 'TX_NONE');
this.tables = this.snapshot.tables;
this.schemas = this.snapshot.schemas;
this.indexes = this.snapshot.indexes;
this.snapshot = null;
}
// ---- 事务快照辅助 ----
deepCloneMapMap(source) {
const clone = new Map();
for (const [k, v] of source) {
const innerClone = new Map();
for (const [ik, iv] of v)
innerClone.set(ik, { ...iv });
clone.set(k, innerClone);
}
return clone;
}
deepCloneIndexes(source) {
const clone = new Map();
for (const [tableName, tableIndexes] of source) {
const tableClone = new Map();
for (const [col, colIndex] of tableIndexes) {
const colClone = new Map();
for (const [val, pkSet] of colIndex)
colClone.set(val, new Set(pkSet));
tableClone.set(col, colClone);
}
clone.set(tableName, tableClone);
}
return clone;
}
// ---- 内部辅助 ----
ensureTable(tableName) {
if (!this.tables.has(tableName))
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
}
getPrimaryKey(schema) {
for (const [name, col] of Object.entries(schema.columns)) {
if (col.primaryKey)
return name;
}
return Object.keys(schema.columns)[0];
}
validateRow(schema, row) {
const validated = {};
for (const [colName, colDef] of Object.entries(schema.columns)) {
let value = row[colName];
if (value === undefined && colDef.default !== undefined)
value = colDef.default;
if (colDef.required && (value === undefined || value === null)) {
throw new DatabaseError(`Column "${colName}" is required in table "${schema.name}"`, 'VALIDATION_ERROR');
}
if (value !== undefined && value !== null)
this.checkType(colName, colDef.type, value);
if (value !== undefined)
validated[colName] = value;
}
return validated;
}
checkType(colName, type, value) {
const jsType = typeof value;
switch (type) {
case 'string':
if (jsType !== 'string')
throw new DatabaseError(`Column "${colName}" expects string, got ${jsType}`, 'TYPE_ERROR');
break;
case 'number':
if (jsType !== 'number')
throw new DatabaseError(`Column "${colName}" expects number, got ${jsType}`, 'TYPE_ERROR');
break;
case 'boolean':
if (jsType !== 'boolean')
throw new DatabaseError(`Column "${colName}" expects boolean, got ${jsType}`, 'TYPE_ERROR');
break;
case 'date':
if (jsType !== 'string' || isNaN(Date.parse(value)))
throw new DatabaseError(`Column "${colName}" expects valid date`, 'TYPE_ERROR');
break;
case 'json':
if (jsType !== 'object')
throw new DatabaseError(`Column "${colName}" expects object/array, got ${jsType}`, 'TYPE_ERROR');
break;
}
}
/** O(1) 唯一性检查:利用哈希索引 */
checkUniqueness(schema, row) {
const tableIndexes = this.indexes.get(schema.name);
if (!tableIndexes)
return;
for (const [colName, colDef] of Object.entries(schema.columns)) {
if (!colDef.unique || row[colName] === undefined || row[colName] === null)
continue;
const colIndex = tableIndexes.get(colName);
if (colIndex && colIndex.has(row[colName])) {
throw new DatabaseError(`Unique constraint violation on column "${colName}" in table "${schema.name}"`, 'UNIQUE_VIOLATION');
}
}
}
/** 索引查找 */
tryIndexLookup(tableName, table, query) {
const tableIndexes = this.indexes.get(tableName);
if (!tableIndexes || !query.where)
return Array.from(table.values());
for (const [col, condition] of Object.entries(query.where)) {
// v0.4.1: 支持 { $eq: value } 形式(SQL 解析器生成的等值条件)走索引
let targetValue;
if (typeof condition !== 'object' || condition === null) {
targetValue = condition;
}
else if ('$eq' in condition && Object.keys(condition).length === 1) {
targetValue = condition.$eq;
}
else {
continue;
}
const colIndex = tableIndexes.get(col);
if (colIndex) {
const pks = colIndex.get(targetValue);
if (pks) {
const result = [];
for (const pk of pks) {
const r = table.get(pk);
if (r)
result.push(r);
}
return result;
}
return [];
}
}
return Array.from(table.values());
}
/** 更新索引 */
updateIndexes(tableName, row, pk) {
const tableIndexes = this.indexes.get(tableName);
if (!tableIndexes)
return;
for (const [colName, colIndex] of tableIndexes) {
const value = row[colName];
if (value !== undefined && value !== null) {
if (!colIndex.has(value))
colIndex.set(value, new Set());
colIndex.get(value).add(pk);
}
}
}
/** v0.3.3: 从所有索引中移除一行的条目(update/delete 前调用,修复索引过期/残留) */
removeIndexEntries(tableName, row, pk) {
const tableIndexes = this.indexes.get(tableName);
if (!tableIndexes)
return;
for (const [colName, colIndex] of tableIndexes) {
const value = row[colName];
if (value !== undefined && value !== null) {
const pks = colIndex.get(value);
if (pks) {
pks.delete(pk);
if (pks.size === 0)
colIndex.delete(value);
}
}
}
}
// ---- 外键级联 ----
/**
* 级联删除:查找引用 tableName.pkValue 的所有表的行并删除。
* v0.6.1-fix: 环路保护(A→B→A 级联环不再无限递归栈溢出,AriaEngine 同语义)。
* @returns 级联删除的行数
*/
async cascadeDelete(tableName, pkValue, _row, visited = new Set()) {
const visitKey = `${tableName}:${pkValue}`;
if (visited.has(visitKey))
return 0;
visited.add(visitKey);
let totalCascade = 0;
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onDelete)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refTableData = this.tables.get(refTableName);
if (!refTableData)
continue;
// 查找所有引用此主键的行
const toDelete = [];
for (const [refPk, refRow] of refTableData) {
if (String(refRow[colName]) === pkValue) {
toDelete.push(refPk);
}
}
// RESTRICT: 存在引用行时禁止删除
if (colDef.onDelete === 'RESTRICT' && toDelete.length > 0) {
throw new DatabaseError(`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
if (colDef.onDelete === 'CASCADE') {
// 递归级联
for (const refPk of toDelete) {
const refRow = refTableData.get(refPk);
if (refRow) {
// v0.3.3: 级联删除前清理索引条目
this.removeIndexEntries(refTableName, refRow, refPk);
totalCascade += await this.cascadeDelete(refTableName, refPk, refRow, visited);
}
refTableData.delete(refPk);
totalCascade++;
}
}
else if (colDef.onDelete === 'SET NULL') {
for (const refPk of toDelete) {
const refRow = refTableData.get(refPk);
if (refRow) {
// v0.6.3-fix: 复用 removeIndexEntries 清理旧值索引 —— 此前手动
// `pks.get(v)?.delete(pk)` 后遗留空 Set → checkUniqueness 对旧值
// 永久误报 UNIQUE_VIOLATION(外键列带 unique 约束时)
this.removeIndexEntries(refTableName, refRow, refPk);
refRow[colName] = null;
this.updateIndexes(refTableName, refRow, refPk);
}
}
}
}
}
return totalCascade;
}
}
/** 崩溃残留临时文件后缀(createWritable 底层实现可能留下) */
const STALE_SUFFIXES = ['.crswap', '.tmp'];
class OPFSBackend {
constructor() {
this.root = null;
this.dbDir = null;
this.dbName = '';
this.writeQueue = Promise.resolve();
}
async open(name) {
this.dbName = name;
this.root = await navigator.storage.getDirectory();
this.dbDir = await this.root.getDirectoryHandle(name, { create: true });
// v0.4.5: 清理崩溃残留临时文件(不阻塞打开)
await this.cleanupStaleFiles();
}
async close() {
// v0.4.5-fix: 等待所有挂起写完成(否则 close 后挂起写被静默丢弃/读旧数据)
try {
await this.writeQueue;
}
catch { /* 写失败已返回给调用方 */ }
this.dbDir = null;
this.root = null;
}
isOpen() {
return this.dbDir !== null;
}
/** 清理崩溃残留的临时文件(open 时自动调用,repair 也可调用) */
async cleanupStaleFiles() {
if (!this.dbDir)
return;
try {
const dir = this.dbDir;
const stale = [];
for await (const [name] of dir.entries()) {
if (STALE_SUFFIXES.some((s) => name.endsWith(s))) {
stale.push(name);
}
}
for (const name of stale) {
try {
await this.dbDir.removeEntry(name);
}
catch { /* ignore */ }
}
}
catch { /* 清理失败不阻塞 */ }
}
async read(key) {
if (!this.dbDir)
return null;
try {
const fh = await this.dbDir.getFileHandle(key);
const file = await fh.getFile();
return await file.arrayBuffer();
}
catch {
return null;
}
}
/**
* 单文件原子写:createWritable 为 copy-on-writeclose 后原子替换;
* 写入期间崩溃 → 旧文件保持(原子性由浏览器 OPFS 实现保证)。
*/
async write(key, data) {
if (!this.dbDir)
return;
const run = this.writeQueue.then(async () => {
const fh = await this.dbDir.getFileHandle(key, { create: true });
const writable = await fh.createWritable();
await writable.write(data);
await writable.close();
});
// v0.4.5-fix: 单次任务失败不中断队列链(错误仍返回给本次调用方)
this.writeQueue = run.then(() => undefined, () => undefined);
return run;
}
/**
* v0.4.5: 真追加写 — createWritable(keepExistingData) + seek 到文件末尾。
* 单文件 COW 原子(close 前崩溃旧文件保持),无需读旧内容即实现 O(chunk) 追加
* (WAL 分片高频写入用)。
*/
async append(key, data) {
if (!this.dbDir)
return;
const run = this.writeQueue.then(async () => {
const fh = await this.dbDir.getFileHandle(key, { create: true });
const existing = await fh.getFile();
const writable = await fh.createWritable({ keepExistingData: true });
await writable.write({ type: 'write', position: existing.size, data });
await writable.close();
});
this.writeQueue = run.then(() => undefined, () => undefined);
return run;
}
/** v0.4.2-fix: 批量写入 — 串行队列内逐个落盘(OPFS 无跨文件事务,顺序保证一致) */
async writeMany(entries) {
if (!this.dbDir)
return;
const run = this.writeQueue.then(async () => {
for (const [key, data] of Object.entries(entries)) {
const fh = await this.dbDir.getFileHandle(key, { create: true });
const writable = await fh.createWritable();
await writable.write(data);
await writable.close();
}
});
this.writeQueue = run.then(() => undefined, () => undefined);
return run;
}
async delete(key) {
if (!this.dbDir)
return;
const run = this.writeQueue.then(async () => {
try {
await this.dbDir.removeEntry(key);
}
catch { /* ignore */ }
});
this.writeQueue = run.then(() => undefined, () => undefined);
return run;
}
/** v0.4.2-fix: 批量删除 — 串行队列内逐个删除 */
async deleteMany(keys) {
if (!this.dbDir)
return;
const run = this.writeQueue.then(async () => {
for (const key of keys) {
try {
await this.dbDir.removeEntry(key);
}
catch { /* ignore */ }
}
});
this.writeQueue = run.then(() => undefined, () => undefined);
return run;
}
async listKeys() {
if (!this.dbDir)
return [];
const keys = [];
// FileSystemDirectoryHandle.entries() 返回 AsyncIterable,使用 any 绕过 dts 生成限制
const dir = this.dbDir;
for await (const [name] of dir.entries()) {
keys.push(name);
}
return keys;
}
async exists(key) {
if (!this.dbDir)
return false;
try {
await this.dbDir.getFileHandle(key);
return true;
}
catch {
return false;
}
}
async clear() {
if (!this.dbDir)
return;
const dir = this.dbDir;
for await (const [name] of dir.entries()) {
try {
await this.dbDir.removeEntry(name);
}
catch { /* ignore */ }
}
}
}
/**
* KVStore SharedMemory Medium — 跨实例共享的内存介质
* @module engine/kvstore/shared_memory_medium
*
* v0.6.0: 替代 fake-indexeddb 的测试/Node 环境介质。
* 与 MemoryBackend 的区别:数据按库名存于全局注册表,close() 不清除
* (模拟"磁盘持久化"语义——重新 open 同名库可读到上次写入的数据)。
*
* 仅用于测试与 Node 环境;浏览器使用 OPFS 介质(KVStore 默认自动选择)。
*/
/** 全局注册表:dbName → key → chunk 列表(跨实例共享,模拟持久化) */
const registry = new Map();
class SharedMemoryBackend {
constructor() {
this.dbName = '';
/** 存储:key → chunk 列表(append O(1)read 时一次性拼接缓存) */
this.chunks = new Map();
/** 惰性拼接缓存(read 后缓存,write/append 失效) */
this.materialized = new Map();
}
/** 清空全局注册表(测试隔离用) */
static clearRegistry() {
registry.clear();
}
async open(name) {
this.dbName = name;
if (!registry.has(name)) {
registry.set(name, new Map());
}
// 从注册表恢复 chunks(持久化语义)
this.chunks = registry.get(name);
this.materialized = new Map();
}
/** close 不清除数据(持久化语义:重开同名库数据仍在) */
async close() {
this.chunks = new Map();
this.materialized = new Map();
}
isOpen() {
return this.dbName !== '';
}
async read(key) {
if (this.materialized.has(key))
return this.materialized.get(key);
const list = this.chunks.get(key);
if (!list || list.length === 0)
return null;
if (list.length === 1) {
this.materialized.set(key, list[0]);
return list[0];
}
const total = list.reduce((s2, c) => s2 + c.byteLength, 0);
const combined = new Uint8Array(total);
let off = 0;
for (const c of list) {
combined.set(new Uint8Array(c), off);
off += c.byteLength;
}
const buf = combined.buffer;
this.materialized.set(key, buf);
return buf;
}
async write(key, data) {
this.chunks.set(key, [data]);
this.materialized.set(key, data);
}
async append(key, data) {
// O(1) 追加:只记录 chunkread 时惰性拼接
const list = this.chunks.get(key);
if (list) {
list.push(data);
}
else {
this.chunks.set(key, [data]);
}
this.materialized.delete(key);
}
async writeMany(entries) {
// 同步批量写入 = 原子(JS 单线程,无中间 await 点)
for (const [key, data] of Object.entries(entries)) {
this.chunks.set(key, [data]);
this.materialized.set(key, data);
}
}
async delete(key) {
this.chunks.delete(key);
this.materialized.delete(key);
}
async deleteMany(keys) {
for (const key of keys) {
this.chunks.delete(key);
this.materialized.delete(key);
}
}
async listKeys() {
return Array.from(this.chunks.keys());
}
async exists(key) {
return this.chunks.has(key);
}
async clear() {
this.chunks.clear();
this.materialized.clear();
}
}
/**
* AriaEngine CRC32 — 标准 CRC-32IEEE 802.3,多项式 0xEDB88320
* @module engine/aria/crc32
*
* 查表法实现。
* 分段计算约定:
* crc32(head + tail) === crc32Finalize(crc32Update(crc32Update(0xFFFFFFFF, head), tail))
* === crc32(tail, crc32(head))
* 用于 SSTable 文件校验和与 WAL 记录完整性校验。
*/
/** CRC-32 查找表(0xEDB88320 反射多项式) */
const CRC32_TABLE = (() => {
const table = new Uint32Array(256);
for (let i = 0; i < 256; i++) {
let c = i;
for (let k = 0; k < 8; k++) {
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
}
table[i] = c >>> 0;
}
return table;
})();
/** CRC-32 初始累加器状态 */
const CRC32_INIT = 0xffffffff;
/**
* 更新内部 CRC 累加状态(供分段计算使用,接收中间状态返回中间状态)。
*/
function crc32Update(state, data) {
let crc = state >>> 0;
for (let i = 0; i < data.byteLength; i++) {
crc = (CRC32_TABLE[(crc ^ data[i]) & 0xff] ^ (crc >>> 8)) >>> 0;
}
return crc >>> 0;
}
/** 将中间状态转换为最终校验和(终止计算) */
function crc32Finalize(state) {
return (state ^ CRC32_INIT) >>> 0;
}
/**
* 计算标准 CRC-32 校验和。
* @param data 输入字节
* @param seed 前序片段计算的最终校验和(首段传 0 或不传)
* @returns 32 位无符号校验和
*/
function crc32(data, seed = 0) {
const state = crc32Update(seed ^ CRC32_INIT, data);
return crc32Finalize(state);
}
/**
* KVStore Log — 追加式事务日志编解码
* @module engine/kvstore/log
*
* v0.6.0: 自研 KV 引擎的原子写载体。
* 每条日志记录 = 一个原子事务(putMany 多键写入 / deleteMany 多键删除)。
* 单文件追加(介质 append,COW 原子)→ 崩溃时记录全有或全无。
*
* 记录格式(大端序):
* [recordLen u32] — 本条记录长度(含自身,不含 CRC)
* [seq u32] — 日志序号(递增,恢复时与快照水位比对去重)
* [entryCount u32] — 条目数
* 每条 entry:
* [op u8] — 1=PUT, 2=DELETE
* [keyLen u32][key bytes]
* [valueLen u32][value bytes] DELETE 时 valueLen=0
* [crc u32] — 覆盖本条记录除 CRC 外全部字节的标准 CRC-32
*/
/** 日志操作类型 */
var KVLogOp;
(function (KVLogOp) {
KVLogOp[KVLogOp["PUT"] = 1] = "PUT";
KVLogOp[KVLogOp["DELETE"] = 2] = "DELETE";
/** v0.6.1: 追加写入(value 拼接语义,恢复时按序 concataria WAL 分片用) */
KVLogOp[KVLogOp["APPEND"] = 3] = "APPEND";
})(KVLogOp || (KVLogOp = {}));
/**
* 编码一条日志记录。
* @param seq 日志序号
* @param puts key → value 写入条目
* @param deletes 删除 key 列表
* @param appends key → 追加块列表(APPEND 语义,恢复时拼接)
*/
function encodeLogRecord(seq, puts, deletes = [], appends = {}) {
const encoder = new TextEncoder();
const entries = [];
for (const [key, value] of Object.entries(puts)) {
entries.push({ op: KVLogOp.PUT, key, value });
}
for (const key of deletes) {
entries.push({ op: KVLogOp.DELETE, key, value: new ArrayBuffer(0) });
}
for (const [key, value] of Object.entries(appends)) {
entries.push({ op: KVLogOp.APPEND, key, value });
}
// 预编码 key 字节,计算总长度
const entryBytes = [];
let total = 4 + 4 + 4; // recordLen + seq + entryCount
for (const e of entries) {
const kb = encoder.encode(e.key);
const vb = new Uint8Array(e.value);
entryBytes.push({ op: e.op, key: kb, value: vb });
total += 1 + 4 + kb.byteLength + 4 + vb.byteLength;
}
total += 4; // crc
const buf = new Uint8Array(total);
const view = new DataView(buf.buffer);
let offset = 0;
view.setUint32(offset, total - 4, false);
offset += 4; // recordLen(不含 CRC
view.setUint32(offset, seq, false);
offset += 4;
view.setUint32(offset, entryBytes.length, false);
offset += 4;
for (const e of entryBytes) {
view.setUint8(offset, e.op);
offset += 1;
view.setUint32(offset, e.key.byteLength, false);
offset += 4;
buf.set(e.key, offset);
offset += e.key.byteLength;
view.setUint32(offset, e.value.byteLength, false);
offset += 4;
buf.set(e.value, offset);
offset += e.value.byteLength;
}
const crc = crc32(buf.subarray(0, total - 4));
view.setUint32(total - 4, crc, false);
return buf;
}
/**
* 解析日志中的全部记录(顺序扫描)。
* @param data 日志字节流
* @param onRecord 每条有效记录回调(CRC 通过)
* @param onCorrupt 损坏记录位置回调(返回 false 停止扫描,或继续尝试下一条)
* @returns 有效记录数
*/
function parseLogRecords(data, onRecord, onCorrupt) {
let offset = 0;
let count = 0;
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
const decoder = new TextDecoder();
while (offset + 4 <= data.byteLength) {
const recordLen = view.getUint32(offset, false);
if (recordLen < 12 || offset + 4 + recordLen > data.byteLength) {
// 尾部残缺记录(最后一批写入被截断):损坏
if (onCorrupt) {
if (!onCorrupt(offset))
break;
}
break;
}
const recordStart = offset;
const recordEnd = offset + 4 + recordLen;
const raw = data.subarray(recordStart, recordEnd);
const recView = new DataView(data.buffer, data.byteOffset + recordStart, recordLen + 4);
const storedCrc = recView.getUint32(recordLen, false);
const computedCrc = crc32(raw.subarray(0, recordLen));
if (storedCrc !== computedCrc) {
if (onCorrupt) {
if (!onCorrupt(recordStart))
break;
}
break;
}
// 解析条目
let p = 4;
const seq = recView.getUint32(p, false);
p += 4;
const entryCount = recView.getUint32(p, false);
p += 4;
const entries = [];
let valid = true;
for (let i = 0; i < entryCount; i++) {
if (p + 1 + 4 > recordLen + 4) {
valid = false;
break;
}
const op = recView.getUint8(p);
p += 1;
const keyLen = recView.getUint32(p, false);
p += 4;
if (p + keyLen + 4 > recordLen + 4) {
valid = false;
break;
}
const key = decoder.decode(raw.subarray(p, p + keyLen));
p += keyLen;
const valueLen = recView.getUint32(p, false);
p += 4;
if (p + valueLen > recordLen + 4) {
valid = false;
break;
}
const value = raw.slice(p, p + valueLen).buffer;
p += valueLen;
entries.push({ op, key, value });
}
if (!valid) {
if (onCorrupt) {
if (!onCorrupt(recordStart))
break;
}
break;
}
onRecord({ seq, entries, raw });
count++;
offset = recordEnd;
}
return count;
}
/**
* KVStore Snapshot — 快照序列化/反序列化
* @module engine/kvstore/snapshot
*
* v0.6.0: checkpoint 时把全部 key-value 序列化为快照文件(COW 原子写),
* 快照内嵌"日志水位 seq"(快照包含的最后一条日志序号),恢复时只重放 seq > 水位 的记录。
*
* 格式(大端序):
* [magic u32] — 0x4B56534E ("KVSN")
* [seq u32] — 日志水位(快照包含的数据对应的日志序号)
* [entryCount u32]
* 每条: [keyLen u32][key bytes][valueLen u32][value bytes]
* [crc u32] — 覆盖除 CRC 外全部字节的标准 CRC-32
*/
const SNAPSHOT_MAGIC = 0x4b56534e; // "KVSN"
/** 序列化快照 */
function encodeSnapshot(seq, entries) {
const encoder = new TextEncoder();
const keys = Array.from(entries.keys());
// 预编码
const encoded = [];
let total = 4 + 4 + 4; // magic + seq + entryCount
for (const key of keys) {
const kb = encoder.encode(key);
const vb = new Uint8Array(entries.get(key));
encoded.push({ key: kb, value: vb });
total += 4 + kb.byteLength + 4 + vb.byteLength;
}
total += 4; // crc
const buf = new Uint8Array(total);
const view = new DataView(buf.buffer);
let offset = 0;
view.setUint32(offset, SNAPSHOT_MAGIC, false);
offset += 4;
view.setUint32(offset, seq, false);
offset += 4;
view.setUint32(offset, encoded.length, false);
offset += 4;
for (const e of encoded) {
view.setUint32(offset, e.key.byteLength, false);
offset += 4;
buf.set(e.key, offset);
offset += e.key.byteLength;
view.setUint32(offset, e.value.byteLength, false);
offset += 4;
buf.set(e.value, offset);
offset += e.value.byteLength;
}
const crc = crc32(buf.subarray(0, total - 4));
view.setUint32(total - 4, crc, false);
return buf;
}
/**
* 解析快照。
* @returns 快照内容;损坏(magic 错误/CRC 失败/越界)返回 null
*/
function decodeSnapshot(data) {
if (data.byteLength < 16)
return null;
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
if (view.getUint32(0, false) !== SNAPSHOT_MAGIC)
return null;
const storedCrc = view.getUint32(data.byteLength - 4, false);
const computedCrc = crc32(data.subarray(0, data.byteLength - 4));
if (storedCrc !== computedCrc)
return null;
const decoder = new TextDecoder();
const entries = new Map();
let p = 4;
const seq = view.getUint32(p, false);
p += 4;
const entryCount = view.getUint32(p, false);
p += 4;
for (let i = 0; i < entryCount; i++) {
if (p + 4 > data.byteLength - 4)
return null;
const keyLen = view.getUint32(p, false);
p += 4;
if (p + keyLen + 4 > data.byteLength - 4)
return null;
const key = decoder.decode(data.subarray(p, p + keyLen));
p += keyLen;
const valueLen = view.getUint32(p, false);
p += 4;
if (p + valueLen > data.byteLength - 4)
return null;
const value = data.slice(p, p + valueLen).buffer;
p += valueLen;
entries.set(key, value);
}
return { seq, entries };
}
/**
* KVStore — 自研 KV 事务存储引擎(替代 IndexedDB)
* @module engine/kvstore/index
*
* v0.6.0: 在浏览器文件系统(OPFS)之上实现 IndexedDB 级能力:
* - 多 key 原子事务:putMany/deleteMany 写入单条日志记录(单文件 COW 原子追加)→
* 崩溃时记录全有或全无(IndexedDB 事务同等的原子性,但完全自研)
* - 持久化与崩溃恢复:快照(checkpoint)+ 追加日志(WAL 式),两阶段恢复
* - 自愈:快照损坏回退全量日志重放;日志损坏截断至损坏处(丢弃未确认尾部)
* - 容错时序:checkpoint = 写快照 → 写 meta → 清空日志(meta 先于截断,
* 任何崩溃窗口数据不丢)
*
* 介质层为 IStorageBackendOPFSBackend / SharedMemoryBackend):
* - 浏览器:自动选择 OPFSnavigator.storage
* - Node/测试:SharedMemoryBackend(跨实例共享,模拟持久化)
*
* 可靠性设计:
* - 所有写操作与 checkpoint 经内部串行队列(快照与日志水位一致,无交错窗口)
* - 日志记录与快照均有标准 CRC-32 校验
* - 内存索引为热路径(get O(1)),checkpoint 后日志截断
*/
/** 存储键 */
const LOG_KEY = '__kv_log';
const SNAPSHOT_KEY = '__kv_snapshot';
const META_KEY = '__kv_meta';
/** checkpoint 自动触发阈值(日志字节数,0=不自动) */
const DEFAULT_CHECKPOINT_THRESHOLD = 16 * 1024 * 1024;
function defaultMedium() {
const nav = globalThis.navigator;
if (typeof nav !== 'undefined' && nav.storage && typeof nav.storage.getDirectory === 'function') {
return new OPFSBackend();
}
return new SharedMemoryBackend();
}
class KVStore {
constructor(medium, checkpointThreshold = DEFAULT_CHECKPOINT_THRESHOLD) {
this.dbName = '';
this.opened = false;
/** 内存索引(热路径权威视图) */
this.index = new Map();
/** 日志水位(最后一条已应用日志记录序号) */
this.seq = 0;
/** 日志累计字节数(checkpoint 阈值) */
this.logBytes = 0;
/** 写操作串行队列(checkpoint 与写入无交错窗口) */
this.opQueue = Promise.resolve();
/** 最近一次后台操作失败(checkpoint 时报告) */
this.lastBackgroundError = null;
this.medium = medium ?? defaultMedium();
this.checkpointThreshold = checkpointThreshold;
}
isOpen() {
return this.opened;
}
// =======================================================================
// 生命周期
// =======================================================================
/** 打开(加载快照 + 重放日志) */
async open(dbName) {
if (this.opened)
return;
this.dbName = dbName;
await this.medium.open(dbName);
this.index = new Map();
this.seq = 0;
this.logBytes = 0;
// 1. 加载快照(损坏则全量日志重放;快照内嵌 seq 为日志水位权威)
// metacheckpoint 写入)仅标记库存在,不参与水位判断
let snapshotSeq = 0;
const snapshotRaw = await this.medium.read(SNAPSHOT_KEY);
if (snapshotRaw) {
const snap = decodeSnapshot(new Uint8Array(snapshotRaw));
if (snap) {
this.index = new Map(snap.entries);
this.seq = snap.seq;
snapshotSeq = snap.seq;
}
else {
// 快照损坏:从空索引 + 全量日志重放
this.index = new Map();
this.seq = 0;
snapshotSeq = 0;
}
}
// 3. 重放日志(seq > 快照水位的记录)
// v0.6.1-fix: 快照水位只信任快照内嵌 seqsnapshotSeq)。
// 此前 baseSeq = max(metaSeq, snapshotSeq):快照损坏回退全量重放时,
// meta.seq(最后一次 checkpoint 水位)会错误跳过日志中 checkpoint 后
// 的有效记录(日志被截断过,metaSeq 不代表日志内容水位)→ 丢数据。
// meta 仅在库存在性标记,不参与水位判断。
const logRaw = await this.medium.read(LOG_KEY);
if (logRaw && logRaw.byteLength > 0) {
const log = new Uint8Array(logRaw);
const baseSeq = snapshotSeq;
const corruptOffsets = [];
const applied = parseLogRecords(log, (record) => {
if (record.seq <= baseSeq)
return; // 快照已包含,跳过(幂等)
this.applyRecord(record.entries);
this.seq = record.seq;
}, (offset) => {
corruptOffsets.push(offset);
return true; // 记录损坏位置后停止(日志是顺序流,无法跳过继续)
});
if (applied > 0 || corruptOffsets.length > 0) {
this.logBytes = log.byteLength;
}
if (corruptOffsets.length > 0) {
// 损坏尾部:截断日志(丢弃未确认记录),下次 checkpoint 落盘
await this.truncateLog();
}
}
this.opened = true;
}
/**
* v0.6.0: 从介质重新加载(多标签页同步/外部写入可见用)。
* KVStore 的内存索引非跨实例共享,重新 open 读取介质最新数据。
*/
async reload() {
if (!this.opened)
return;
try {
await this.opQueue;
}
catch { /* ignore */ }
this.index = new Map();
this.seq = 0;
this.logBytes = 0;
this.opened = false;
await this.open(this.dbName);
}
/** 关闭(不丢弃数据;下次 open 同名库恢复) */
async close() {
if (!this.opened)
return;
// 排空写队列
try {
await this.opQueue;
}
catch { /* 写失败已返回 */ }
await this.medium.close();
this.index.clear();
this.seq = 0;
this.logBytes = 0;
this.opened = false;
}
// =======================================================================
// 读写(内存热路径)
// =======================================================================
async get(key) {
return this.index.get(key) ?? null;
}
async getAll() {
return Array.from(this.index.entries());
}
async listKeys() {
return Array.from(this.index.keys());
}
async exists(key) {
return this.index.has(key);
}
size() {
return this.index.size;
}
// =======================================================================
// 写入(原子事务)
// =======================================================================
/** 单 key 写入(原子) */
async put(key, value) {
await this.enqueue(async () => {
await this.appendRecord({ [key]: value }, []);
});
}
/** 多 key 原子写入(单条日志记录,崩溃全有或全无) */
async putMany(entries) {
if (Object.keys(entries).length === 0)
return;
await this.enqueue(async () => {
await this.appendRecord(entries, []);
});
}
/** 单 key 删除(原子) */
async delete(key) {
await this.enqueue(async () => {
await this.appendRecord({}, [key]);
});
}
/** 多 key 原子删除(单条日志记录) */
async deleteMany(keys) {
if (keys.length === 0)
return;
await this.enqueue(async () => {
await this.appendRecord({}, keys);
});
}
/**
* v0.6.3: 多 key 混合原子写(put + delete 编码进同一条日志记录)。
* 此前 KVStoreEngine 的 delete/update 主键变更等路径 putMany 与 deleteMany
* 分两次调用 = 两条记录:崩溃在两条记录之间 → 新旧行并存(重复行/脏数据),
* 与"一条日志记录 = 真原子"宣称不符。此方法保证混合操作全有或全无。
*/
async writeBatch(puts, deletes) {
if (Object.keys(puts).length === 0 && deletes.length === 0)
return;
await this.enqueue(async () => {
await this.appendRecord(puts, deletes);
});
}
/**
* v0.6.1: 追加写入(value 拼接语义)— aria WAL 分片等追加型数据用。
* 日志记录 APPEND 类型(O(chunk) 高效),恢复时按 seq 顺序拼接,
* checkpoint 后快照含最终值。崩溃时该次追加全有或全无(单记录原子)。
*/
async appendValue(key, chunk) {
if (chunk.byteLength === 0)
return;
await this.enqueue(async () => {
await this.appendRecord({}, [], { [key]: chunk });
});
}
// =======================================================================
// 维护
// =======================================================================
/** checkpoint:快照 → meta → 截断日志(时序保证任何崩溃窗口不丢数据) */
async checkpoint() {
await this.enqueue(async () => {
// 报告上次后台失败
if (this.lastBackgroundError !== null) {
const error = this.lastBackgroundError;
this.lastBackgroundError = null;
throw new DatabaseError('KVStore background write failed', 'KV_BACKGROUND_ERROR', error);
}
if (this.logBytes === 0 && this.index.size === 0)
return;
// 1. 写快照(COW 原子)
const snapBytes = encodeSnapshot(this.seq, this.index);
await this.medium.write(SNAPSHOT_KEY, snapBytes.buffer);
// 2. 写 meta(指向新水位)
const meta = { seq: this.seq };
await this.medium.write(META_KEY, new TextEncoder().encode(JSON.stringify(meta)).buffer);
// 3. 截断日志(meta 已更新 → 截断安全)
await this.truncateLog();
});
}
/** 清空全部数据(保留库本身) */
async clear() {
await this.enqueue(async () => {
await this.medium.clear();
this.index.clear();
this.seq = 0;
this.logBytes = 0;
// 写空 meta(下次 open 正常初始化)
const meta = { seq: 0 };
await this.medium.write(META_KEY, new TextEncoder().encode(JSON.stringify(meta)).buffer);
});
}
/**
* 自愈:校验快照/日志完整性,清理损坏数据。
* @returns 丢弃的损坏日志字节数(0 = 无损坏)
*/
async repair() {
return this.enqueue(async () => {
let discarded = 0;
// 1. 校验快照:损坏则删除(下次打开全量日志重放)
const snapRaw = await this.medium.read(SNAPSHOT_KEY);
if (snapRaw && !decodeSnapshot(new Uint8Array(snapRaw))) {
await this.medium.delete(SNAPSHOT_KEY);
discarded++;
}
// 2. 校验日志:损坏尾部截断
const logRaw = await this.medium.read(LOG_KEY);
if (logRaw && logRaw.byteLength > 0) {
const log = new Uint8Array(logRaw);
const validBytes = this.findValidLogLength(log);
if (validBytes < log.byteLength) {
discarded += log.byteLength - validBytes;
const truncated = log.subarray(0, validBytes).slice().buffer;
await this.medium.write(LOG_KEY, truncated);
}
}
return discarded;
});
}
// =======================================================================
// 内部
// =======================================================================
enqueue(fn) {
const run = this.opQueue.then(fn, fn);
this.opQueue = run.then(() => undefined, () => undefined);
return run;
}
/** 追加一条日志记录并更新内存索引(队列内调用,无并发) */
async appendRecord(puts, deletes, appends = {}) {
this.seq++;
const record = encodeLogRecord(this.seq, puts, deletes, appends);
try {
// 日志追加:介质 append(真追加)或回退读+拼+写
const data = record.buffer.slice(record.byteOffset, record.byteOffset + record.byteLength);
if (typeof this.medium.append === 'function') {
await this.medium.append(LOG_KEY, data);
}
else {
const existing = await this.medium.read(LOG_KEY);
if (existing) {
const combined = new Uint8Array(existing.byteLength + data.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(data), existing.byteLength);
await this.medium.write(LOG_KEY, combined.buffer);
}
else {
await this.medium.write(LOG_KEY, data);
}
}
}
catch (error) {
// 记录写入失败:内存索引不更新(原子性),记录后台错误
this.seq--; // 回滚水位
this.lastBackgroundError = error;
throw new DatabaseError('KVStore log append failed', 'KV_LOG_ERROR', error);
}
// 日志成功:更新内存索引(原子语义)
for (const [key, value] of Object.entries(puts)) {
this.index.set(key, value);
}
for (const key of deletes) {
this.index.delete(key);
}
for (const [key, chunk] of Object.entries(appends)) {
const existing = this.index.get(key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + chunk.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(chunk), existing.byteLength);
this.index.set(key, combined.buffer);
}
else {
this.index.set(key, chunk);
}
}
this.logBytes += record.byteLength;
// 自动 checkpoint(日志超阈值)
if (this.checkpointThreshold > 0 && this.logBytes >= this.checkpointThreshold) {
await this.medium.write(SNAPSHOT_KEY, encodeSnapshot(this.seq, this.index).buffer);
const meta = { seq: this.seq };
await this.medium.write(META_KEY, new TextEncoder().encode(JSON.stringify(meta)).buffer);
await this.truncateLog();
}
}
/** 截断日志(清空文件) */
async truncateLog() {
try {
await this.medium.write(LOG_KEY, new ArrayBuffer(0));
}
catch { /* 截断失败:下次 checkpoint 重试 */ }
this.logBytes = 0;
}
/** 应用记录条目到内存索引 */
applyRecord(entries) {
for (const e of entries) {
if (e.op === KVLogOp.PUT) {
this.index.set(e.key, e.value);
}
else if (e.op === KVLogOp.APPEND) {
const existing = this.index.get(e.key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + e.value.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(e.value), existing.byteLength);
this.index.set(e.key, combined.buffer);
}
else {
this.index.set(e.key, e.value);
}
}
else {
this.index.delete(e.key);
}
}
}
/** 确定日志中有效字节长度(从 0 开始连续解析到第一条损坏/残缺记录) */
findValidLogLength(log) {
let offset = 0;
const view = new DataView(log.buffer, log.byteOffset, log.byteLength);
while (offset + 4 <= log.byteLength) {
const recordLen = view.getUint32(offset, false);
if (recordLen < 12 || offset + 4 + recordLen > log.byteLength)
break;
const raw = log.subarray(offset, offset + 4 + recordLen);
const storedCrc = view.getUint32(offset + recordLen, false);
if (crc32(raw.subarray(0, recordLen)) !== storedCrc)
break;
offset += 4 + recordLen;
}
return offset;
}
}
/**
* KVStoreEngine — 基于自研 KVStore 的磁盘存储引擎(替代 IndexedDBEngine / OPFSEngine
* @module engine/kvstore_engine
*
* v0.6.0: 完全移除 IndexedDB 后的 disk 模式引擎。
*
* 架构:MemoryEngine(内存热路径 + 事务快照)+ KVStore(持久化 + 原子写)
* - 读:始终走内存(写路径同步落盘,重启从 KVStore 恢复)
* - 写:内存先行 + KVStore 增量持久化(insert 增量 putManyupdate/delete 受影响行重写;
* 主键变更/级联场景整表 diff;全部原子)
* - 事务:内存快照 + commit 时受影响表原子 flushwriteBatch 单记录 = 真原子,
* 此前 IndexedDBEngine 依赖 IDB 事务,现在完全自研)
*
* 数据布局(KVStore keys:
* `__schema` — JSON { tableName: TableSchema }
* `__meta:{key}` — 库内元数据(迁移版本等)
* `t:{table}:{pk}` — 行数据(JSON
*/
const SCHEMA_KEY = '__schema';
const ROW_PREFIX = 't:';
const enc = (s) => new TextEncoder().encode(s).buffer;
const dec = (b) => new TextDecoder().decode(b);
class KVStoreEngine {
constructor(medium, checkpointThreshold) {
this.name = 'kv';
this.memory = new MemoryEngine();
this.dbName = '';
this.version = 1;
this.opened = false;
/** 活跃事务标记 */
this.txActive = false;
/** 事务中写过的表(commit 时只 flush 这些表) */
this.txDirtyTables = new Set();
/** 事务中发生 schema 变更(DDL)—— commit 时持久化 schema */
this.txSchemaChanged = false;
/**
* v0.7.0: 事务行级变更记录(table → pk → put/delete)。
* commit 时按行增量 flush(此前整表 diff:大表事务改 1 行也重写全表)。
*/
this.txChanges = new Map();
/** v0.7.0: 无法行级追踪的表(主键变更/级联影响表)→ commit 时整表 diff */
this.txFullTables = new Set();
/** v0.7.0: 事务内 clear 的表 → commit 时清空 KV 行 */
this.txClearedTables = new Set();
this.kv = new KVStore(medium, checkpointThreshold);
}
// ---- 行 key 编解码 ----
rowKey(table, pk) {
return `${ROW_PREFIX}${table}:${pk}`;
}
rowPrefix(table) {
return `${ROW_PREFIX}${table}:`;
}
// ---- 生命周期 ----
async open(dbName, version) {
if (this.opened)
return;
this.dbName = dbName;
this.version = version;
await this.kv.open(dbName);
await this.memory.open(dbName, version);
// 恢复 schema
const schemaRaw = await this.kv.get(SCHEMA_KEY);
if (schemaRaw) {
try {
const schemas = JSON.parse(dec(schemaRaw));
for (const schema of Object.values(schemas)) {
await this.memory.createTable(schema);
}
}
catch {
throw new DatabaseError('Corrupted schema in KVStore', 'KV_SCHEMA_ERROR');
}
}
// 恢复行数据 + 重建索引
const all = await this.kv.getAll();
for (const [key, value] of all) {
if (!key.startsWith(ROW_PREFIX))
continue;
const sep = key.indexOf(':', ROW_PREFIX.length);
if (sep < 0)
continue;
const table = key.slice(ROW_PREFIX.length, sep);
if (!(await this.memory.hasTable(table)))
continue;
try {
const row = JSON.parse(dec(value));
await this.memory.insert(table, [row]);
}
catch {
// 单行损坏跳过(repair 可清理)
}
}
// 重建二级索引(schema 标记的索引列)
const tables = await this.memory.getTableNames();
for (const table of tables) {
const schema = await this.memory.getTableSchema(table);
if (!schema)
continue;
for (const [col, colDef] of Object.entries(schema.columns)) {
if (colDef.index || colDef.unique) {
await this.memory.createIndex(table, col, colDef.unique);
}
}
}
this.opened = true;
}
async close() {
if (!this.opened)
return;
// 活跃事务先回滚
if (this.txActive) {
try {
await this.rollbackTransaction();
}
catch { /* ignore */ }
}
// v0.7.0: 关闭前 checkpoint(截断日志,重开更快)。
// 失败不阻塞关闭(日志已持久,重开可全量重放)。
try {
await this.kv.checkpoint();
}
catch { /* 数据在日志中,重开放心重放 */ }
await this.kv.close();
await this.memory.close();
this.opened = false;
}
isOpen() {
return this.opened;
}
/**
* v0.6.0: 从 KVStore 重新加载全部数据到内存(多标签页同步重载用)。
* Hybrid 引擎的 reloadMemoryFromDisk 依赖磁盘引擎"读穿透"
* KVStoreEngine 读内存 → 提供 reload 重新加载磁盘最新数据。
*/
async reload() {
if (!this.opened)
return;
// 1. KVStore 重新从介质加载(外部写入可见)
await this.kv.reload();
// 2. 内存缓存重载
await this.memory.close();
await this.memory.open(this.dbName, this.version);
this.opened = false;
await this.open(this.dbName, this.version);
}
/** v0.4.2-fix: 自愈 — 校验 KVStore 日志/快照完整性并重建内存 */
async repair() {
this.ensureOpen();
await this.kv.repair();
await this.memory.close();
await this.memory.open(this.dbName, this.version);
// 重新恢复(复用 open 的恢复逻辑)
this.opened = false;
await this.open(this.dbName, this.version);
}
async clearAll() {
this.ensureOpen();
await this.kv.clear();
await this.memory.clearAll();
}
async getMeta(key) {
const raw = await this.kv.get(`__meta:${key}`);
return raw ? dec(raw) : null;
}
async setMeta(key, value) {
await this.kv.put(`__meta:${key}`, enc(value));
}
// ---- 表管理 ----
async createTable(schema) {
this.ensureOpen();
await this.memory.createTable(schema);
if (this.txActive) {
this.txDirtyTables.add(schema.name);
this.txSchemaChanged = true;
return;
}
await this.persistSchema();
}
async dropTable(tableName) {
this.ensureOpen();
await this.memory.dropTable(tableName);
if (this.txActive) {
this.txDirtyTables.add(tableName);
this.txSchemaChanged = true;
return;
}
await this.persistSchema();
// 删除该表全部行(KV 中残留清理)
const diff = await this.collectTableDiff(tableName);
await this.kv.writeBatch(diff.puts, diff.deletes);
}
async hasTable(tableName) {
this.ensureOpen();
return this.memory.hasTable(tableName);
}
async getTableNames() {
this.ensureOpen();
return this.memory.getTableNames();
}
async getTableSchema(tableName) {
this.ensureOpen();
return this.memory.getTableSchema(tableName);
}
async alterTable(tableName, action, column) {
this.ensureOpen();
// v0.7.2: 事务内 ALTER 显式拒绝(与 AriaEngine/MemoryEngine 对齐)——
// memory.alterTable 直接修改共享 columns 对象,事务快照无法回滚
// (此前 ROLLBACK 后新增列残留)
if (this.txActive) {
throw new DatabaseError(`ALTER TABLE is not supported inside a transaction (KVStoreEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
await this.memory.alterTable(tableName, action, column);
if (this.txActive) {
this.txDirtyTables.add(tableName);
this.txSchemaChanged = true;
return;
}
await this.persistSchema();
if (action === 'DROP') {
// 重写存储行(移除该列)
const diff = await this.collectTableDiff(tableName);
await this.kv.writeBatch(diff.puts, diff.deletes);
}
}
// ---- CRUD ----
async insert(tableName, rows) {
this.ensureOpen();
const pks = await this.memory.insert(tableName, rows);
if (this.txActive) {
this.txDirtyTables.add(tableName);
// v0.7.0: 事务内 clear 后又写入 → 清空语义被覆盖,整表 diff 兜底
if (this.txClearedTables.has(tableName)) {
this.txClearedTables.delete(tableName);
this.txFullTables.add(tableName);
return pks;
}
// v0.7.0: 行级变更记录(增量 flush)
let changes = this.txChanges.get(tableName);
if (!changes) {
changes = new Map();
this.txChanges.set(tableName, changes);
}
for (const pk of pks)
changes.set(pk, 'put');
return pks;
}
// 增量持久化(原子 putMany
const schema = await this.memory.getTableSchema(tableName);
if (!schema)
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
const pkCol = this.getPK(schema);
const puts = {};
rows.forEach((row, i) => {
puts[this.rowKey(tableName, String(pks[i] ?? row[pkCol]))] = enc(JSON.stringify(row));
});
await this.kv.putMany(puts);
return pks;
}
async find(tableName, query) {
this.ensureOpen();
return this.memory.find(tableName, query);
}
async findStream(tableName, query, onRow) {
this.ensureOpen();
return this.memory.findStream(tableName, query, onRow);
}
async update(tableName, query, updates) {
this.ensureOpen();
const schema = await this.memory.getTableSchema(tableName);
if (!schema)
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
const pkCol = this.getPK(schema);
// v0.7.2: undefined 值视为"不更新该列"(与 memory.update 语义对齐)
const cleanUpdates = stripUndefinedUpdates(updates);
const pkChanged = pkCol in cleanUpdates;
// 收集受影响旧主键(内存匹配)
const affected = pkChanged ? [] : await this.collectMatchingPks(tableName, query);
const count = await this.memory.update(tableName, query, cleanUpdates);
if (this.txActive) {
this.txDirtyTables.add(tableName);
// v0.7.0: 行级变更记录 —— 主键变更无法行级追踪(旧键删除+新键落盘+级联),
// 相关表整表 diff 兜底;普通更新记录受影响行
if (pkChanged) {
for (const t of await this.affectedTables(tableName)) {
this.txFullTables.add(t);
this.txDirtyTables.add(t);
}
}
else {
let changes = this.txChanges.get(tableName);
if (!changes) {
changes = new Map();
this.txChanges.set(tableName, changes);
}
for (const pk of affected)
changes.set(pk, 'put');
// 级联影响表(理论上非主键更新不级联,防御性兜底)
for (const t of await this.affectedTables(tableName)) {
if (t !== tableName) {
this.txFullTables.add(t);
this.txDirtyTables.add(t);
}
}
}
return count;
}
const puts = {};
const deletes = [];
if (pkChanged) {
// 主键变更:相关表整表 diff(罕见操作,可靠性优先)
for (const t of await this.affectedTables(tableName)) {
const diff = await this.collectTableDiff(t);
Object.assign(puts, diff.puts);
deletes.push(...diff.deletes);
}
}
else {
// v0.6.2-fix(P0): 受影响主键经 String() 化后按 `where { [pkCol]: pk }` 回查内存行,
// 数值型主键(123 !== "123")不命中 → 行被误判删除 → 重启丢数据。
// 改为单次全表扫描 + 受影响集合过滤(同时消除此前 O(N×M) 逐主键回查开销)。
const affectedSet = new Set(affected);
const allRows = await this.memory.find(tableName, { table: tableName });
for (const row of allRows) {
const pkStr = String(row[pkCol]);
if (affectedSet.has(pkStr)) {
puts[this.rowKey(tableName, pkStr)] = enc(JSON.stringify(row));
affectedSet.delete(pkStr);
}
}
// 剩余主键(内存中已不存在,如被级联移除)→ 删除对应 KV 行
for (const pk of affectedSet) {
deletes.push(this.rowKey(tableName, pk));
}
// 级联影响表(SET NULL/CASCADE 外键)整表 diff
for (const t of await this.affectedTables(tableName)) {
if (t === tableName)
continue;
const diff = await this.collectTableDiff(t);
Object.assign(puts, diff.puts);
deletes.push(...diff.deletes);
}
}
// 单次原子写(一条日志记录 = 真原子,v0.6.1)
await this.kv.writeBatch(puts, deletes);
return count;
}
async delete(tableName, query) {
this.ensureOpen();
// 收集受影响主键(内存匹配)
const pks = await this.collectMatchingPks(tableName, query);
const count = await this.memory.delete(tableName, query);
if (this.txActive) {
this.txDirtyTables.add(tableName);
// v0.7.0: 行级变更记录 —— 删除行记录 delete;级联影响表整表 diff 兜底
let changes = this.txChanges.get(tableName);
if (!changes) {
changes = new Map();
this.txChanges.set(tableName, changes);
}
for (const pk of pks)
changes.set(pk, 'delete');
for (const t of await this.affectedTables(tableName)) {
if (t !== tableName) {
this.txFullTables.add(t);
this.txDirtyTables.add(t);
}
}
return count;
}
const puts = {};
const deletes = pks.map((pk) => this.rowKey(tableName, pk));
// 级联影响表整表 diff(合并到单次原子写,v0.6.1)
for (const t of await this.affectedTables(tableName)) {
if (t === tableName)
continue;
const diff = await this.collectTableDiff(t);
Object.assign(puts, diff.puts);
deletes.push(...diff.deletes);
}
await this.kv.writeBatch(puts, deletes);
return count;
}
async count(tableName, query) {
this.ensureOpen();
return this.memory.count(tableName, query);
}
async clear(tableName) {
this.ensureOpen();
await this.memory.clear(tableName);
if (this.txActive) {
this.txDirtyTables.add(tableName);
// v0.7.0: 事务内清空 → commit 时删除全部 KV 行(比整表 diff 更高效)
this.txClearedTables.add(tableName);
this.txChanges.delete(tableName);
return;
}
const diff = await this.collectTableDiff(tableName);
await this.kv.writeBatch(diff.puts, diff.deletes);
}
// ---- 动态索引 ----
async createIndex(tableName, column, unique) {
this.ensureOpen();
if (this.txActive) {
throw new DatabaseError(`CREATE INDEX is not supported inside a transaction (KVStoreEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
await this.memory.createIndex(tableName, column, unique);
if (this.txActive) {
this.txDirtyTables.add(tableName);
this.txSchemaChanged = true;
return;
}
await this.persistSchema();
}
async dropIndex(tableName, column, indexName) {
this.ensureOpen();
if (this.txActive) {
throw new DatabaseError(`DROP INDEX is not supported inside a transaction (KVStoreEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
await this.memory.dropIndex(tableName, column, indexName);
if (this.txActive) {
this.txDirtyTables.add(tableName);
this.txSchemaChanged = true;
return;
}
await this.persistSchema();
}
// ---- 事务(原子 flush ----
async beginTransaction() {
this.ensureOpen();
await this.memory.beginTransaction();
this.txActive = true;
this.txDirtyTables = new Set();
this.txSchemaChanged = false;
this.txChanges = new Map();
this.txFullTables = new Set();
this.txClearedTables = new Set();
}
async commitTransaction() {
this.ensureOpen();
if (!this.txActive)
throw new DatabaseError('No active transaction', 'TX_NONE');
// v0.6.1: 全部 dirty 表合并为单次原子 flush(一条日志记录 = 真原子,
// 多表事务中途崩溃/失败不会出现"部分表已提交")。
// v0.7.0: 行级增量 flush —— 普通 insert/update/delete 仅写事务内改动的行
// (此前 collectTableDiff 整表重写:大表事务改 1 行也 O(表大小));
// 主键变更/级联影响表整表 diff 兜底;clear/drop 表只删 KV 行。
const puts = {};
const deletes = [];
for (const table of this.txDirtyTables) {
if (!(await this.memory.hasTable(table))) {
// 事务内 drop 的表:清理 KV 残留行
const all = await this.kv.getAll();
const prefix = this.rowPrefix(table);
for (const [key] of all) {
if (key.startsWith(prefix))
deletes.push(key);
}
continue;
}
if (this.txClearedTables.has(table)) {
// 事务内 clear 的表:删除全部 KV 行
const all = await this.kv.getAll();
const prefix = this.rowPrefix(table);
for (const [key] of all) {
if (key.startsWith(prefix))
deletes.push(key);
}
continue;
}
if (this.txFullTables.has(table)) {
const diff = await this.collectTableDiff(table);
Object.assign(puts, diff.puts);
deletes.push(...diff.deletes);
continue;
}
const changes = this.txChanges.get(table);
if (changes && changes.size > 0) {
// v0.7.0: 增量 flush —— 单次全表扫描 + 变更集合过滤
const schema = await this.memory.getTableSchema(table);
if (!schema)
continue;
const pkCol = this.getPK(schema);
const pending = new Map(changes);
const rows = await this.memory.find(table, { table: table });
for (const row of rows) {
const pkStr = String(row[pkCol]);
const kind = pending.get(pkStr);
if (kind !== undefined) {
if (kind === 'put')
puts[this.rowKey(table, pkStr)] = enc(JSON.stringify(row));
else
deletes.push(this.rowKey(table, pkStr));
pending.delete(pkStr);
}
}
// 内存中已不存在的行(后续操作删除)→ KV 行删除
for (const [pk, kind] of pending) {
if (kind === 'delete')
deletes.push(this.rowKey(table, pk));
}
}
}
await this.kv.writeBatch(puts, deletes);
// 事务内 DDL 的 schema 一并持久化
if (this.txSchemaChanged) {
await this.persistSchema();
}
// v0.7.0-perf: 移除每次 commit 的强制全量 checkpoint —— KVStore 按日志阈值
// 自动 checkpoint(日志重放保证崩溃恢复正确),大库高频事务不再 O(库大小)。
// close() 时统一 checkpoint(截断日志,重开更快)。
await this.memory.commitTransaction();
this.txActive = false;
this.txDirtyTables = new Set();
this.txChanges = new Map();
this.txFullTables = new Set();
this.txClearedTables = new Set();
}
async rollbackTransaction() {
this.ensureOpen();
if (!this.txActive)
throw new DatabaseError('No active transaction', 'TX_NONE');
await this.memory.rollbackTransaction();
this.txActive = false;
this.txDirtyTables = new Set();
this.txSchemaChanged = false;
this.txChanges = new Map();
this.txFullTables = new Set();
this.txClearedTables = new Set();
}
// ---- 内部 ----
ensureOpen() {
if (!this.opened)
throw new DatabaseError('Database not opened', 'DB_NOT_OPEN');
}
getPK(schema) {
for (const [name, col] of Object.entries(schema.columns)) {
if (col.primaryKey)
return name;
}
return Object.keys(schema.columns)[0];
}
/** 收集匹配查询的内存行主键(持久化差异计算用) */
async collectMatchingPks(tableName, query) {
const schema = await this.memory.getTableSchema(tableName);
if (!schema)
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
const pkCol = this.getPK(schema);
const rows = await this.memory.find(tableName, query);
return rows.map((r) => String(r[pkCol]));
}
/**
* 计算外键级联影响的表集合(传递闭包:A 被 B 引用,B 被 C 引用 → {A, B, C})。
* 级联操作(delete/update 主键)需要把这些表一并重写持久化。
*/
async affectedTables(tableName) {
const set = new Set([tableName]);
let changed = true;
while (changed) {
changed = false;
for (const table of await this.memory.getTableNames()) {
if (set.has(table))
continue;
const schema = await this.memory.getTableSchema(table);
if (!schema)
continue;
for (const col of Object.values(schema.columns)) {
if (col.references) {
const ref = col.references.split('.')[0];
if (set.has(ref)) {
set.add(table);
changed = true;
break;
}
}
}
}
}
return set;
}
/** 持久化 schema(全部表) */
async persistSchema() {
const schemas = {};
for (const table of await this.memory.getTableNames()) {
const schema = await this.memory.getTableSchema(table);
if (schema)
schemas[table] = schema;
}
await this.kv.put(SCHEMA_KEY, enc(JSON.stringify(schemas)));
}
/**
* v0.6.1: 整表 diff 收集(不落盘):内存行全部 put + KV 残留行删除。
* 调用方合并到单次原子 writeBatchput+delete 同一条日志记录,多表操作真原子)。
*/
async collectTableDiff(tableName) {
const prefix = this.rowPrefix(tableName);
const puts = {};
const deletes = [];
// 表已删除:仅收集 KV 残留行删除
const schema = await this.memory.getTableSchema(tableName);
if (schema) {
const pkCol = this.getPK(schema);
const rows = await this.memory.find(tableName, { table: tableName });
const current = new Set();
for (const row of rows) {
const key = this.rowKey(tableName, String(row[pkCol]));
current.add(key);
puts[key] = enc(JSON.stringify(row));
}
// KV 残留行(内存中已不存在)
const all = await this.kv.getAll();
for (const [key] of all) {
if (key.startsWith(prefix) && !current.has(key))
deletes.push(key);
}
}
else {
const all = await this.kv.getAll();
for (const [key] of all) {
if (key.startsWith(prefix))
deletes.push(key);
}
}
return { puts, deletes };
}
}
/**
* AriaEngine Types — 内部类型定义
* @module engine/aria/types
*
* 页面式存储引擎的所有内部枚举、接口和常量。
*/
// =============================================================================
// 页面常量
// =============================================================================
/** 页面大小:4KB */
const PAGE_SIZE = 4096;
/** 页面头大小:16 字节 */
const PAGE_HEADER_SIZE = 16;
// =============================================================================
// 页面类型
// =============================================================================
var PageType;
(function (PageType) {
/** 数据页面:存储 SSTable 字节切片 */
PageType[PageType["DATA"] = 1] = "DATA";
/** 索引页面:存储索引节点 */
PageType[PageType["INDEX"] = 2] = "INDEX";
/** 溢出页面:存储大字段 */
PageType[PageType["OVERFLOW"] = 3] = "OVERFLOW";
/** 元数据页面:存储表/库元信息 */
PageType[PageType["META"] = 4] = "META";
})(PageType || (PageType = {}));
// =============================================================================
// 列类型(内部二进制编码用)
// =============================================================================
// =============================================================================
// LSM-Tree
// =============================================================================
/** MemTable 最大大小(默认 4MB */
const DEFAULT_MEMTABLE_SIZE = 4 * 1024 * 1024;
/** Bloom Filter 每 key 的默认位数 */
const DEFAULT_BLOOM_BITS_PER_KEY = 10;
/** SSTable 最大层级 */
const MAX_LSM_LEVELS = 7;
/** 每层之间的大小倍数 */
const DEFAULT_LEVEL_SIZE_MULTIPLIER = 10;
// =============================================================================
// WAL (Write-Ahead Log)
// =============================================================================
/** WAL 记录类型 */
var WALRecordType;
(function (WALRecordType) {
WALRecordType[WALRecordType["INSERT"] = 1] = "INSERT";
WALRecordType[WALRecordType["UPDATE"] = 2] = "UPDATE";
WALRecordType[WALRecordType["DELETE"] = 3] = "DELETE";
WALRecordType[WALRecordType["BEGIN"] = 4] = "BEGIN";
WALRecordType[WALRecordType["COMMIT"] = 5] = "COMMIT";
WALRecordType[WALRecordType["ROLLBACK"] = 6] = "ROLLBACK";
WALRecordType[WALRecordType["CREATE_TABLE"] = 7] = "CREATE_TABLE";
WALRecordType[WALRecordType["DROP_TABLE"] = 8] = "DROP_TABLE";
})(WALRecordType || (WALRecordType = {}));
// =============================================================================
// MVCC
// =============================================================================
/** 事务状态 */
var TransactionState;
(function (TransactionState) {
TransactionState[TransactionState["ACTIVE"] = 1] = "ACTIVE";
TransactionState[TransactionState["COMMITTED"] = 2] = "COMMITTED";
TransactionState[TransactionState["ABORTED"] = 3] = "ABORTED";
})(TransactionState || (TransactionState = {}));
// =============================================================================
// Buffer Pool
// =============================================================================
/** Buffer Pool 默认容量:256 页 ≈ 1MB */
const DEFAULT_BUFFER_POOL_PAGES = 256;
const DEFAULT_ARIA_CONFIG = {
pageSize: PAGE_SIZE,
bufferPoolPages: DEFAULT_BUFFER_POOL_PAGES,
memtableSizeThreshold: DEFAULT_MEMTABLE_SIZE,
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
walEnabled: true,
walSyncMode: 'full',
checkpointInterval: 1000,
compression: false,
storageBackend: 'opfs',
walSizeThreshold: 16 * 1024 * 1024, // 16MB
maxMemoryMB: 64,
encryption: undefined,
pageStorage: undefined,
};
/**
* AriaEngine MemTable — 基于红黑树的内存表
* @module engine/aria/index/memtable
*
* 写操作先进入 MemTable,达到阈值后冻结并 flush 成 SSTable。
*/
// ---------------------------------------------------------------------------
// RB-Tree Node
// ---------------------------------------------------------------------------
var Color;
(function (Color) {
Color[Color["RED"] = 0] = "RED";
Color[Color["BLACK"] = 1] = "BLACK";
})(Color || (Color = {}));
class RBNode {
constructor(key, value) {
this.color = Color.RED;
this.left = null;
this.right = null;
this.parent = null;
this.key = key;
this.value = value;
}
}
// ---------------------------------------------------------------------------
// Red-Black Tree
// ---------------------------------------------------------------------------
class RedBlackTree {
constructor() {
this.root = null;
this._size = 0;
}
get size() { return this._size; }
// ---- 插入 ----
insert(key, value) {
const node = new RBNode(key, value);
if (!this.root) {
this.root = node;
node.color = Color.BLACK;
this._size++;
return;
}
let parent = null;
let current = this.root;
while (current) {
parent = current;
if (key < current.key) {
current = current.left;
}
else if (key > current.key) {
current = current.right;
}
else {
// 更新已存在的 key
current.value = value;
return;
}
}
node.parent = parent;
if (key < parent.key) {
parent.left = node;
}
else {
parent.right = node;
}
this._size++;
this.fixInsert(node);
}
// ---- 查找 ----
find(key) {
let current = this.root;
while (current) {
if (key < current.key) {
current = current.left;
}
else if (key > current.key) {
current = current.right;
}
else {
return current.value;
}
}
return null;
}
// ---- 删除 ----
delete(key) {
// 简化实现:标记删除(实际改为找到并调整树)
const node = this.findNode(key);
if (!node)
return false;
this.deleteNode(node);
this._size--;
return true;
}
// ---- 遍历 ----
/** 中序遍历(有序) */
inorder(callback) {
this._inorder(this.root, callback);
}
/** 范围遍历 */
rangeScan(startKey, endKey, callback) {
this._rangeScan(this.root, startKey, endKey, callback);
}
/** 获取所有条目 */
getAllEntries() {
const entries = [];
this.inorder((k, v) => entries.push([k, v]));
return entries;
}
/** 清空 */
clear() {
this.root = null;
this._size = 0;
}
// ---- 内部方法 ----
findNode(key) {
let current = this.root;
while (current) {
if (key < current.key) {
current = current.left;
}
else if (key > current.key) {
current = current.right;
}
else {
return current;
}
}
return null;
}
deleteNode(node) {
// 简化:用左子树最大或右子树最小替换
// 完整实现较复杂,这里采用简化策略
if (!node.left && !node.right) {
this.transplant(node, null);
if (node.color === Color.BLACK)
this.fixDelete(null, node.parent);
}
else if (!node.left) {
this.transplant(node, node.right);
if (node.color === Color.BLACK)
this.fixDelete(node.right, node.right.parent);
}
else if (!node.right) {
this.transplant(node, node.left);
if (node.color === Color.BLACK)
this.fixDelete(node.left, node.left.parent);
}
else {
const successor = this.minimum(node.right);
if (successor.parent !== node) {
this.transplant(successor, successor.right);
successor.right = node.right;
successor.right.parent = successor;
}
this.transplant(node, successor);
successor.left = node.left;
successor.left.parent = successor;
const origColor = successor.color;
successor.color = node.color;
if (origColor === Color.BLACK)
this.fixDelete(successor.right, successor.right?.parent ?? null);
}
}
transplant(u, v) {
if (!u.parent) {
this.root = v;
}
else if (u === u.parent.left) {
u.parent.left = v;
}
else {
u.parent.right = v;
}
if (v)
v.parent = u.parent;
}
minimum(node) {
while (node.left)
node = node.left;
return node;
}
fixInsert(node) {
while (node.parent && node.parent.color === Color.RED) {
const parent = node.parent;
const grandparent = parent.parent;
if (!grandparent)
break;
if (parent === grandparent.left) {
const uncle = grandparent.right;
if (uncle && uncle.color === Color.RED) {
parent.color = Color.BLACK;
uncle.color = Color.BLACK;
grandparent.color = Color.RED;
node = grandparent;
}
else {
if (node === parent.right) {
node = parent;
this.rotateLeft(node);
}
if (node.parent)
node.parent.color = Color.BLACK;
if (node.parent?.parent)
node.parent.parent.color = Color.RED;
if (node.parent?.parent)
this.rotateRight(node.parent.parent);
}
}
else {
const uncle = grandparent.left;
if (uncle && uncle.color === Color.RED) {
parent.color = Color.BLACK;
uncle.color = Color.BLACK;
grandparent.color = Color.RED;
node = grandparent;
}
else {
if (node === parent.left) {
node = parent;
this.rotateRight(node);
}
if (node.parent)
node.parent.color = Color.BLACK;
if (node.parent?.parent)
node.parent.parent.color = Color.RED;
if (node.parent?.parent)
this.rotateLeft(node.parent.parent);
}
}
}
if (this.root)
this.root.color = Color.BLACK;
}
fixDelete(x, parent) {
// 标准 RB-Tree 删除修复(修复"双黑"问题)
let node = x;
let nodeParent = parent;
while ((!node || node.color === Color.BLACK) && node !== this.root) {
if (!nodeParent)
break;
if (node === nodeParent.left) {
let sibling = nodeParent.right;
if (!sibling)
break;
// Case 1: 兄弟是红色
if (sibling.color === Color.RED) {
sibling.color = Color.BLACK;
nodeParent.color = Color.RED;
this.rotateLeft(nodeParent);
sibling = nodeParent.right;
if (!sibling)
break;
}
// Case 2: 兄弟的两个子节点都是黑色
const sibLeft = sibling.left;
const sibRight = sibling.right;
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
(!sibRight || sibRight.color === Color.BLACK)) {
sibling.color = Color.RED;
node = nodeParent;
nodeParent = node.parent;
}
else {
// Case 3: 兄弟右子黑色(左子红色)
if (!sibRight || sibRight.color === Color.BLACK) {
if (sibLeft)
sibLeft.color = Color.BLACK;
sibling.color = Color.RED;
this.rotateRight(sibling);
sibling = nodeParent.right;
if (!sibling)
break;
}
// Case 4: 兄弟右子红色
sibling.color = nodeParent.color;
nodeParent.color = Color.BLACK;
if (sibling.right)
sibling.right.color = Color.BLACK;
this.rotateLeft(nodeParent);
node = this.root;
}
}
else {
// 镜像:node 是父节点的右子
let sibling = nodeParent.left;
if (!sibling)
break;
if (sibling.color === Color.RED) {
sibling.color = Color.BLACK;
nodeParent.color = Color.RED;
this.rotateRight(nodeParent);
sibling = nodeParent.left;
if (!sibling)
break;
}
const sibLeft = sibling.left;
const sibRight = sibling.right;
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
(!sibRight || sibRight.color === Color.BLACK)) {
sibling.color = Color.RED;
node = nodeParent;
nodeParent = node.parent;
}
else {
if (!sibLeft || sibLeft.color === Color.BLACK) {
if (sibRight)
sibRight.color = Color.BLACK;
sibling.color = Color.RED;
this.rotateLeft(sibling);
sibling = nodeParent.left;
if (!sibling)
break;
}
sibling.color = nodeParent.color;
nodeParent.color = Color.BLACK;
if (sibling.left)
sibling.left.color = Color.BLACK;
this.rotateRight(nodeParent);
node = this.root;
}
}
}
if (node)
node.color = Color.BLACK;
}
rotateLeft(x) {
const y = x.right;
if (!y)
return;
x.right = y.left;
if (y.left)
y.left.parent = x;
y.parent = x.parent;
if (!x.parent) {
this.root = y;
}
else if (x === x.parent.left) {
x.parent.left = y;
}
else {
x.parent.right = y;
}
y.left = x;
x.parent = y;
}
rotateRight(x) {
const y = x.left;
if (!y)
return;
x.left = y.right;
if (y.right)
y.right.parent = x;
y.parent = x.parent;
if (!x.parent) {
this.root = y;
}
else if (x === x.parent.right) {
x.parent.right = y;
}
else {
x.parent.left = y;
}
y.right = x;
x.parent = y;
}
_inorder(node, cb) {
if (!node)
return;
this._inorder(node.left, cb);
cb(node.key, node.value);
this._inorder(node.right, cb);
}
_rangeScan(node, start, end, cb) {
if (!node)
return;
if (node.key > start)
this._rangeScan(node.left, start, end, cb);
if (node.key >= start && node.key <= end)
cb(node.key, node.value);
if (node.key < end)
this._rangeScan(node.right, start, end, cb);
}
}
// ---------------------------------------------------------------------------
// MemTable
// ---------------------------------------------------------------------------
class MemTable {
constructor(maxSize = 4 * 1024 * 1024) {
this._estimatedSize = 0;
this.tree = new RedBlackTree();
this.maxSize = maxSize;
}
/** 插入或更新 */
put(key, value) {
const oldSize = this.estimateEntrySize(key, this.tree.find(key));
const newSize = this.estimateEntrySize(key, value);
this.tree.insert(key, value);
this._estimatedSize += newSize - oldSize;
}
/** 获取 */
get(key) {
return this.tree.find(key);
}
/** 删除 */
delete(key) {
const oldVal = this.tree.find(key);
if (oldVal) {
this._estimatedSize -= this.estimateEntrySize(key, oldVal);
}
return this.tree.delete(key);
}
/** 是否应刷盘 */
shouldFlush() {
return this._estimatedSize >= this.maxSize;
}
/** 获取所有有序条目 */
getAllEntries() {
return this.tree.getAllEntries();
}
/** 范围扫描 */
rangeScan(startKey, endKey) {
const entries = [];
this.tree.rangeScan(startKey, endKey, (k, v) => entries.push([k, v]));
return entries;
}
/** 条目数 */
getEntryCount() {
return this.tree.size;
}
/** 估计大小(字节) */
getEstimatedSize() {
return this._estimatedSize;
}
/** 清空 */
clear() {
this.tree.clear();
this._estimatedSize = 0;
}
// -----------------------------------------------------------------------
// 内部
// -----------------------------------------------------------------------
estimateEntrySize(key, value) {
if (!value)
return 0;
let size = key.length * 2; // UTF-16
for (const entry of Object.entries(value)) {
size += entry[0].length * 2;
const v = entry[1];
if (typeof v === 'string')
size += v.length * 2;
else if (typeof v === 'number')
size += 8;
else if (typeof v === 'boolean')
size += 1;
else if (v === null || v === undefined)
size += 1;
else
size += 16; // rough estimate
}
return size;
}
}
/**
* AriaEngine Bloom Filter — 快速判定 key 是否可能存在
* @module engine/aria/index/bloom
*
* 使用双哈希函数 + Kirsch-Mitzenmacher 优化生成 k 个哈希值。
*/
// ---------------------------------------------------------------------------
// BloomFilter
// ---------------------------------------------------------------------------
class BloomFilter {
/**
* @param numKeys 预期插入的 key 数量
* @param bitsPerKey 每个 key 的位数(默认 10,误报率约 1%)
*/
constructor(numKeys, bitsPerKey = DEFAULT_BLOOM_BITS_PER_KEY) {
this._inserted = 0;
// ceil(numKeys * bitsPerKey / 8),最少 64 位
const numBits = Math.max(64, numKeys * bitsPerKey);
const numBytes = Math.ceil(numBits / 8);
this.bits = new Uint8Array(numBytes);
// k = bitsPerKey * ln(2) ≈ bitsPerKey * 0.69
this.numHashes = Math.max(1, Math.floor(bitsPerKey * 0.69));
}
/** 从现有数据恢复 */
static fromData(data, numHashes) {
const bf = new BloomFilter(1); // dummy
bf.bits = data;
bf.numHashes = numHashes;
return bf;
}
/** 插入 key */
insert(key) {
const hashes = this.getHashes(key);
for (const h of hashes) {
const byteIdx = Math.floor(h / 8);
const bitIdx = h % 8;
this.bits[byteIdx] |= (1 << bitIdx);
}
this._inserted++;
}
/** 检查 key 可能存在(false positive 可能,false negative 不可能) */
mayContain(key) {
const hashes = this.getHashes(key);
for (const h of hashes) {
const byteIdx = Math.floor(h / 8);
const bitIdx = h % 8;
if ((this.bits[byteIdx] & (1 << bitIdx)) === 0) {
return false; // 确定不存在
}
}
return true; // 可能存在
}
/** 获取序列化数据 */
serialize() {
return this.bits;
}
/** hash 函数数量 */
getHashCount() {
return this.numHashes;
}
// -----------------------------------------------------------------------
// 哈希
// -----------------------------------------------------------------------
getHashes(key) {
const bits = this.bits.byteLength * 8;
const h1 = this.fnv1a(key);
const h2 = this.murmurSimple(key);
const hashes = [];
for (let i = 0; i < this.numHashes; i++) {
// Kirsch-Mitzenmacher: h_i = h1 + i * h2
const h = Math.abs((h1 + i * h2) % bits);
hashes.push(h);
}
return hashes;
}
/** FNV-1a 哈希 */
fnv1a(str) {
let hash = 0x811c9dc5;
for (let i = 0; i < str.length; i++) {
hash ^= str.charCodeAt(i);
hash = (hash * 0x01000193) >>> 0;
}
return hash;
}
/** 简化的 Murmur-like 哈希 */
murmurSimple(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
const ch = str.charCodeAt(i);
hash = ((hash << 5) - hash + ch) | 0;
hash = (hash ^ (hash >>> 16)) >>> 0;
}
return Math.abs(hash);
}
}
/**
* AriaEngine SSTable Builder — 构建有序字符串表
* @module engine/aria/index/sstable_builder
*
* 将排序后的 key-value 数据写入 SSTable 格式。
*
* v0.4.4 格式 v2magic "SSTC")修复:
* - 块大小估算改用 UTF-8 字节长度(TextEncoder 预编码),
* 此前用字符串 .length(UTF-16 码元)估算而实际写入 UTF-8 字节,
* 中文内容(1 字 3 字节)导致缓冲区低估 → 写入越界崩溃
* - keyLen/valueLen 从 u16 升级为 u32(此前 >64KB 的 value 长度被截断,
* 线性格式整体错乱)
* - 大 value 单条独立成块(切分逻辑基于字节估算)
*
* SSTable 文件布局 (v2):
* ┌──────────────────────────────────────────────┐
* │ Data Block 0 │
* │ Data Block 1 │
* │ ... │
* │ Index Block (block offset → key range) │
* │ Bloom Filter │
* │ Footer (32 bytes) │
* │ - index_offset (u32) │
* │ - index_size (u32) │
* │ - bloom_offset (u32) │
* │ - bloom_size (u32) │
* │ - bloom_hash_count (u32) │
* │ - entry_count (u32) │
* │ - magic_number (u32, 0x53535443 ="SSTC")│
* │ - checksum (u32) │
* └──────────────────────────────────────────────┘
* 数据块条目: entryCount(u32) + [keyLen(u32) + key + valueLen(u32) + value]
* 索引块条目: [keyLen(u32) + key + blockOffset(u32) + blockSize(u32)]
*/
/** v1 格式魔数("SSTB",u16 长度字段,兼容旧文件读取) */
const SSTABLE_MAGIC_V1 = 0x53535442;
/** v2 格式魔数("SSTC"u32 长度字段 + 字节精确估算,v0.4.4) */
const SSTABLE_MAGIC_V2 = 0x53535443;
const SSTABLE_FOOTER_SIZE = 32;
class SSTableBuilder {
constructor(blockSizeLimit = 4096) {
this.entries = [];
this.blockSizeLimit = blockSizeLimit;
}
/** 添加一个 key-value 条目(必须按键排序添加) */
add(key, value) {
this.entries.push([key, value]);
}
/**
* 构建 SSTable 文件的二进制数据(v2 格式)。
* 返回 { data: Uint8Array, indexEntries: IndexEntry[] }
*/
build() {
// v0.4.4-fix: 预编码全部条目 — 块大小估算必须基于 UTF-8 字节长度,
// 字符串 .length 是 UTF-16 码元(中文 1 字 3 字节 vs 1 码元)→ 缓冲区低估越界
const encoder = new TextEncoder();
const encoded = this.entries.map(([key, value]) => ({
key,
keyBytes: encoder.encode(key),
valueBytes: encoder.encode(JSON.stringify(value)),
}));
const blocks = this.splitIntoBlocks(encoded);
const bloomFilter = new BloomFilter(this.entries.length);
// 预计算总大小(字节)
let totalSize = 0;
const blockOffsets = [];
for (const block of blocks) {
blockOffsets.push(totalSize);
totalSize += this.computeBlockSize(block);
}
// 索引块(块内最后一个 key 作为索引键)
const indexEntries = [];
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
indexEntries.push({
key: block[block.length - 1].key,
blockOffset: blockOffsets[i],
blockSize: this.computeBlockSize(block),
});
}
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
// 序列化 bloom filter 以获取其大小
const bloomData = bloomFilter.serialize();
const bloomSize = bloomData.byteLength;
// 写入到 buffer(包含 bloom block
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
const buf = new ArrayBuffer(finalSize);
const view = new DataView(buf);
let offset = 0;
// ---- Data Blocks ----
for (const block of blocks) {
offset = this.writeDataBlock(view, offset, block, bloomFilter);
}
// ---- Index Block ----
const indexOffset = offset;
offset = this.writeIndexBlock(view, offset, indexEntries);
// ---- Bloom Filter Block ----
const bloomOffset = offset;
new Uint8Array(view.buffer).set(bloomData, offset);
offset += bloomSize;
// ---- Footer ----
const footerOffset = offset;
view.setUint32(footerOffset, indexOffset, false); // index_offset
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
view.setUint32(footerOffset + 20, this.entries.length, false);
view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false);
// checksum 字段先写 0,全部字节就绪后计算整文件 CRC32 再回填
view.setUint32(footerOffset + 28, 0, false);
// v0.4.5: 真实 CRC-32 校验和 — 覆盖除自身(最后 4 字节)外的全部内容。
// checksum 永远非 0(计算结果为 0 时用 1 代替),读取端以 0 识别旧版无校验文件
const all = new Uint8Array(buf);
let checksum = crc32(all.subarray(0, all.byteLength - 4));
if (checksum === 0)
checksum = 1;
view.setUint32(footerOffset + 28, checksum, false);
return {
sstableData: new Uint8Array(buf),
indexEntries,
};
}
/** 获取条目数 */
getEntryCount() {
return this.entries.length;
}
// -----------------------------------------------------------------------
// 内部
// -----------------------------------------------------------------------
/** 按 UTF-8 字节大小切分数据块;大 value 单条独立成块 */
splitIntoBlocks(encoded) {
const blocks = [];
let current = [];
for (const entry of encoded) {
current.push(entry);
// v0.4.4-fix: 基于字节估算;单条超大条目(length===1)独立成块不强行切分
if (this.computeBlockSize(current) >= this.blockSizeLimit && current.length > 1) {
blocks.push(current.slice(0, -1));
current = [entry];
}
}
if (current.length > 0)
blocks.push(current);
return blocks;
}
/** 块字节大小:entryCount(u32) + 每对 [keyLen(u32) + key + valueLen(u32) + value] */
computeBlockSize(block) {
let size = 4;
for (const e of block) {
size += 4 + e.keyBytes.length + 4 + e.valueBytes.length;
}
return size;
}
writeDataBlock(view, offset, block, bloomFilter) {
// entry count
view.setUint32(offset, block.length, false);
offset += 4;
for (const e of block) {
// v0.4.4-fix: 长度字段 u32(此前 u16 截断 >64KB 的 value
if (e.keyBytes.length > 0xFFFFFFFF || e.valueBytes.length > 0xFFFFFFFF) {
throw new Error('SSTable entry too large (exceeds u32 length field)');
}
view.setUint32(offset, e.keyBytes.length, false);
offset += 4;
new Uint8Array(view.buffer).set(e.keyBytes, offset);
offset += e.keyBytes.length;
view.setUint32(offset, e.valueBytes.length, false);
offset += 4;
new Uint8Array(view.buffer).set(e.valueBytes, offset);
offset += e.valueBytes.length;
// 插入 bloom filter
bloomFilter.insert(e.key);
}
return offset;
}
estimateIndexBlockSize(entries) {
// entryCount(u32) + each: keyLen(u32)+key+blockOffset(u32)+blockSize(u32)
let size = 4;
const encoder = new TextEncoder();
for (const entry of entries) {
size += 4 + encoder.encode(entry.key).byteLength + 8;
}
return size;
}
writeIndexBlock(view, offset, entries) {
view.setUint32(offset, entries.length, false);
offset += 4;
for (const entry of entries) {
const encoder = new TextEncoder();
const keyBytes = encoder.encode(entry.key);
view.setUint32(offset, keyBytes.length, false);
offset += 4;
new Uint8Array(view.buffer).set(keyBytes, offset);
offset += keyBytes.length;
view.setUint32(offset, entry.blockOffset, false);
offset += 4;
view.setUint32(offset, entry.blockSize, false);
offset += 4;
}
return offset;
}
}
/**
* AriaEngine SSTable Reader — 从 SSTable 二进制数据中读取
* @module engine/aria/index/sstable
*
* v0.4.4: 支持 v1"SSTB"u16 长度字段)与 v2"SSTC"u32 长度字段)双格式,
* 旧库 v1 文件仍可读(小 value 场景无缺陷),新写入使用 v2。
*/
// ---------------------------------------------------------------------------
// SSTableReader
// ---------------------------------------------------------------------------
class SSTableReader {
constructor(data, meta) {
this.indexEntries = [];
this.entryCount = 0;
this.bloomFilter = null;
/** 格式版本:1 = u16 长度字段(旧),2 = u32 长度字段(v0.4.4 */
this.format = 2;
/** footer 中存储的 checksum0 = 旧版无校验文件) */
this.storedChecksum = 0;
this.data = data;
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
this.meta = meta;
this.parseFooter();
}
/**
* 校验整文件 CRC-32(覆盖除 checksum 字段外的全部字节)。
* checksum === 0 表示旧版文件(v1 / v0.4.4 及更早的 v2),跳过校验返回 true(兼容)。
*/
verifyChecksum() {
if (this.storedChecksum === 0)
return true;
if (this.data.byteLength < 4)
return false;
const computed = crc32(this.data.subarray(0, this.data.byteLength - 4));
return computed === this.storedChecksum;
}
/** 长度字段宽度:v2 = 4 字节 u32v1 = 2 字节 u16 */
lenFieldSize() {
return this.format === 2 ? 4 : 2;
}
readLen(offset) {
return this.format === 2 ? this.view.getUint32(offset, false) : this.view.getUint16(offset, false);
}
// -----------------------------------------------------------------------
// 查询
// -----------------------------------------------------------------------
/** 精确查找 key */
get(targetKey) {
// Bloom Filter 快速否定
if (this.bloomFilter && !this.bloomFilter.mayContain(targetKey))
return null;
const blockIdx = this.locateBlock(targetKey);
if (blockIdx < 0)
return null;
const entry = this.indexEntries[blockIdx];
const blockData = this.getBlockData(entry);
// v0.4.1-fix: 残缺文件(meta 偏移超出实际长度)跳过该块,而非抛 RangeError
if (!blockData)
return null;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
const lenSize = this.lenFieldSize();
const entryCount = blockView.getUint32(0, false);
let offset = 4;
// 顺序扫描 block 内的条目(生产中应二分查找)
for (let i = 0; i < entryCount; i++) {
if (offset + lenSize > blockData.byteLength)
break;
const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength)
break;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
offset += keyLen;
const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + valLen > blockData.byteLength)
break;
const valBytes = blockData.slice(offset, offset + valLen);
offset += valLen;
if (key === targetKey) {
try {
return JSON.parse(new TextDecoder().decode(valBytes));
}
catch {
return null;
}
}
}
return null;
}
/** 范围扫描 */
rangeScan(startKey, endKey, callback) {
if (this.indexEntries.length === 0)
return;
const startBlockIdx = Math.max(0, this.locateBlockGE(startKey));
// v0.6.1-fix(P0): 索引键是"块内最后一个 key"builder 约定),
// locateBlockLE 返回最后一个 tail <= endKey 的块,但下一个块(tail > endKey
// 可能仍包含 < endKey 的条目(如末块 t5:k49425..k49995 尾 key 是 t6 前缀)→
// 被排除导致范围扫描漏读尾部数据。多扫一个块,条目级过滤保证不丢。
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey) + 1);
if (startBlockIdx < 0 || endBlockIdx < 0 || startBlockIdx > endBlockIdx)
return;
const lenSize = this.lenFieldSize();
for (let bi = startBlockIdx; bi <= endBlockIdx && bi >= 0; bi++) {
const entry = this.indexEntries[bi];
const blockData = this.getBlockData(entry);
// v0.4.1-fix: 残缺块跳过(rangeScan 继续后续块,不抛异常)
if (!blockData)
continue;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
const blockEntryCount = blockView.getUint32(0, false);
let offset = 4;
for (let i = 0; i < blockEntryCount; i++) {
if (offset + lenSize > blockData.byteLength)
break;
const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength)
break;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
offset += keyLen;
const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + valLen > blockData.byteLength)
break;
const valBytes = blockData.slice(offset, offset + valLen);
offset += valLen;
if (key >= startKey && key <= endKey) {
try {
const value = JSON.parse(new TextDecoder().decode(valBytes));
callback(key, value);
}
catch {
// skip corrupted entry
}
}
}
}
}
/** 扫描所有条目 */
scanAll(callback) {
const lenSize = this.lenFieldSize();
for (const entry of this.indexEntries) {
const blockData = this.getBlockData(entry);
// v0.4.1-fix: 残缺块跳过(scanAll 继续后续块,不抛异常)
if (!blockData)
continue;
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
const blockEntryCount = blockView.getUint32(0, false);
let offset = 4;
for (let i = 0; i < blockEntryCount; i++) {
if (offset + lenSize > blockData.byteLength)
break;
const keyLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + keyLen + lenSize > blockData.byteLength)
break;
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
offset += keyLen;
const valLen = this.format === 2 ? blockView.getUint32(offset, false) : blockView.getUint16(offset, false);
offset += lenSize;
if (offset + valLen > blockData.byteLength)
break;
const valBytes = blockData.slice(offset, offset + valLen);
offset += valLen;
try {
const value = JSON.parse(new TextDecoder().decode(valBytes));
callback(key, value);
}
catch {
// skip corrupted entry
}
}
}
}
// -----------------------------------------------------------------------
// 内部
// -----------------------------------------------------------------------
parseFooter() {
if (this.data.byteLength < 32) {
throw new Error('SSTable too small: missing footer');
}
const footerOffset = this.data.byteLength - 32;
// 验证魔数(v1 "SSTB" / v2 "SSTC"
const magic = this.view.getUint32(footerOffset + 24, false);
if (magic === SSTABLE_MAGIC_V1) {
this.format = 1;
}
else if (magic === SSTABLE_MAGIC_V2) {
this.format = 2;
}
else {
throw new Error(`Invalid SSTable magic: expected ${SSTABLE_MAGIC_V1} or ${SSTABLE_MAGIC_V2}, got ${magic}`);
}
const indexOffset = this.view.getUint32(footerOffset, false);
const indexSize = this.view.getUint32(footerOffset + 4, false);
const bloomOffset = this.view.getUint32(footerOffset + 8, false);
const bloomSize = this.view.getUint32(footerOffset + 12, false);
const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
this.entryCount = this.view.getUint32(footerOffset + 20, false);
this.storedChecksum = this.view.getUint32(footerOffset + 28, false);
// v0.4.1-fix: 完整性校验 — 索引块必须完全落在文件内,否则视为残缺文件跳过
if (indexOffset + 4 > this.data.byteLength || indexOffset + indexSize > this.data.byteLength) {
return; // 残缺文件:无索引块可读,get/rangeScan 均返回空
}
// 解析索引块
this.parseIndexBlock(indexOffset, indexSize);
// 加载 Bloom Filter
if (bloomOffset > 0 && bloomSize > 0 && bloomOffset + bloomSize <= this.data.byteLength) {
try {
const bloomBytes = this.data.slice(bloomOffset, bloomOffset + bloomSize);
this.bloomFilter = BloomFilter.fromData(bloomBytes, bloomHashCount || 10);
}
catch {
// 损坏的 bloom filter 不影响读取(仅跳过快速否定优化)
}
}
}
parseIndexBlock(offset, _size) {
const entryCount = this.view.getUint32(offset, false);
offset += 4;
const lenSize = this.lenFieldSize();
for (let i = 0; i < entryCount; i++) {
// v0.4.1-fix: 索引条目越界(keyLen/blockOffset/blockSize 超过文件长度)时中止解析,
// 已解析的有效条目仍可用于查询
if (offset + lenSize > this.data.byteLength)
break;
const keyLen = this.readLen(offset);
offset += lenSize;
if (offset + keyLen + 8 > this.data.byteLength)
break;
const key = new TextDecoder().decode(this.data.slice(offset, offset + keyLen));
offset += keyLen;
const blockOffset = this.view.getUint32(offset, false);
offset += 4;
const blockSize = this.view.getUint32(offset, false);
offset += 4;
// 跳过指向文件外的块(残缺写入产物),不抛异常
if (blockSize === 0 || blockOffset + blockSize > this.data.byteLength)
continue;
this.indexEntries.push({ key, blockOffset, blockSize });
}
}
/**
* v0.4.1-fix: 获取索引条目对应的数据块。
* 块偏移/大小越界(残缺 SSTable)时返回 null,由调用方跳过而非抛 RangeError。
*/
getBlockData(entry) {
if (entry.blockSize <= 0 || entry.blockOffset < 0)
return null;
if (entry.blockOffset + entry.blockSize > this.data.byteLength)
return null;
return new Uint8Array(this.data.buffer, this.data.byteOffset + entry.blockOffset, entry.blockSize);
}
/** 二分查找某 key 所在的 block 索引 */
locateBlock(key) {
let lo = 0;
let hi = this.indexEntries.length - 1;
while (lo <= hi) {
const mid = Math.floor((lo + hi) / 2);
const entry = this.indexEntries[mid];
if (key <= entry.key) {
// 检查是否在此 block 范围内
const firstKey = mid === 0 ? '' : this.indexEntries[mid - 1].key;
if (key > firstKey)
return mid;
hi = mid - 1;
}
else {
lo = mid + 1;
}
}
return -1;
}
locateBlockGE(key) {
let lo = 0, hi = this.indexEntries.length;
while (lo < hi) {
const mid = (lo + hi) >> 1;
if (this.indexEntries[mid].key < key)
lo = mid + 1;
else
hi = mid;
}
return lo < this.indexEntries.length ? lo : this.indexEntries.length - 1;
}
locateBlockLE(key) {
let lo = 0, hi = this.indexEntries.length;
while (lo < hi) {
const mid = (lo + hi) >> 1;
if (this.indexEntries[mid].key <= key)
lo = mid + 1;
else
hi = mid;
}
return lo > 0 ? lo - 1 : 0;
}
}
/**
* AriaEngine Merge Iterator — 多路归并迭代器
* @module engine/aria/index/merge_iterator
*
* 对多个有序 SSTable 或 MemTable 的结果进行归并去重(保留最新值)。
*/
/** 数组数据源的迭代器 */
class ArrayEntrySource {
constructor(entries) {
this.index = 0;
this.entries = entries;
}
next() {
if (this.index >= this.entries.length)
return null;
return this.entries[this.index++];
}
reset() {
this.index = 0;
}
}
/** 最小堆 */
class MinHeap {
constructor() {
this.heap = [];
}
push(node) {
this.heap.push(node);
this.bubbleUp(this.heap.length - 1);
}
pop() {
if (this.heap.length === 0)
return null;
if (this.heap.length === 1)
return this.heap.pop();
const result = this.heap[0];
this.heap[0] = this.heap.pop();
this.bubbleDown(0);
return result;
}
peek() {
return this.heap.length > 0 ? this.heap[0] : null;
}
get size() {
return this.heap.length;
}
bubbleUp(idx) {
while (idx > 0) {
const parent = Math.floor((idx - 1) / 2);
if (this.heap[idx].key >= this.heap[parent].key)
break;
[this.heap[idx], this.heap[parent]] = [this.heap[parent], this.heap[idx]];
idx = parent;
}
}
bubbleDown(idx) {
const n = this.heap.length;
while (true) {
let smallest = idx;
const left = 2 * idx + 1;
const right = 2 * idx + 2;
if (left < n && this.heap[left].key < this.heap[smallest].key)
smallest = left;
if (right < n && this.heap[right].key < this.heap[smallest].key)
smallest = right;
if (smallest === idx)
break;
[this.heap[idx], this.heap[smallest]] = [this.heap[smallest], this.heap[idx]];
idx = smallest;
}
}
}
// ---------------------------------------------------------------------------
// MergeIterator
// ---------------------------------------------------------------------------
/**
* 对多个有序数据源进行归并,重复 key 保留最新(后出现的)。
* 数据源按新鲜度排序:越新的数据源在下标越小(如 MemTable 在 SSTable 之前)。
*/
class MergeIterator {
constructor() {
this.sources = [];
this.heap = new MinHeap();
}
/** 添加数据源 */
addSource(source) {
this.sources.push(source);
this.seedFromSource(this.sources.length - 1);
}
/** 获取下一个归并后的条目 */
next() {
if (this.heap.size === 0)
return null;
const first = this.heap.pop();
const key = first.key;
let best = first;
// 刷新 first 来源的下一个值
this.seedFromSource(first.sourceIndex);
// 跳过重复 key:在多个来源中保留 sourceIndex 最小(最新)的条目
while (this.heap.peek() && this.heap.peek().key === key) {
const dup = this.heap.pop();
this.seedFromSource(dup.sourceIndex);
if (dup.sourceIndex < best.sourceIndex) {
best = dup;
}
}
return [best.key, best.value];
}
/** 耗尽管道,返回所有归并结果 */
drain() {
const result = [];
let entry = this.next();
while (entry) {
result.push(entry);
entry = this.next();
}
return result;
}
seedFromSource(sourceIndex) {
const entry = this.sources[sourceIndex].next();
if (entry) {
this.heap.push({
key: entry[0],
value: entry[1],
sourceIndex,
});
}
}
}
/**
* AriaEngine LSM-Tree — 日志结构合并树
* @module engine/aria/index/lsm
*
* 管理 MemTable + 多级 SSTable 的读写和 Compaction。
*
* v0.2.1: 完整持久化 — SSTable 元数据和数据均存入存储后端,
* 启动时自动扫描并加载所有 SSTable。
*/
// ---------------------------------------------------------------------------
// LSM
// ---------------------------------------------------------------------------
class LSM {
constructor(config) {
this.immutableMemtable = null;
/** 所有 pending flush 的 frozen memtable(含 immutableMemtable,旧→新) */
this.frozenMemtables = [];
this.levels = [];
this.sstableCache = new Map();
this.cacheSize = 0;
this.operationCount = 0;
this.initialized = false;
this.compacting = false; // 防止重复触发 compaction
/** 串行化 flush/compaction 链:保证持久化顺序与 id 分配顺序一致 */
this.flushChain = Promise.resolve();
/**
* v0.4.3-fix: 最近一次后台 flush/compaction 失败。
* 后台失败不卡死链(吞错防死锁),但在显式 flush()/close() 时报告(不静默)。
*/
this.lastBackgroundError = null;
this.memtableSizeThreshold = config.memtableSizeThreshold ?? DEFAULT_MEMTABLE_SIZE;
this.memtable = new MemTable(this.memtableSizeThreshold);
this.levelSizeMultiplier = config.levelSizeMultiplier ?? DEFAULT_LEVEL_SIZE_MULTIPLIER;
this.blockSize = config.blockSize ?? 4096;
this.sstableStore = config.sstableStore;
this.cacheLimitBytes = config.cacheLimitBytes ?? 64 * 1024 * 1024;
for (let i = 0; i < MAX_LSM_LEVELS; i++) {
this.levels.push([]);
}
}
// =======================================================================
// 初始化:从存储后端加载 SSTable 元数据
// =======================================================================
async init() {
if (this.initialized)
return;
const metas = await this.sstableStore.listMeta();
// v0.4.2-fix: 打开时完整性校验 — 验证每个 meta 引用的文件存在、可解析,
// 残缺/损坏的 SSTable 忽略并清理 meta,避免后续读取抛 RangeError 崩溃
const validMetas = [];
for (const meta of metas) {
if (await this.validateSSTable(meta)) {
validMetas.push(meta);
}
}
// 按层级分组
for (const meta of validMetas) {
if (meta.level >= 0 && meta.level < MAX_LSM_LEVELS) {
this.levels[meta.level].push(meta);
}
}
// 各层级按 id 降序排列:id 越大越新,保证读取时"最新优先"
// flush/compaction 产物均插入数组头部,数组头部即最新)
for (let i = 0; i < MAX_LSM_LEVELS; i++) {
this.levels[i].sort((a, b) => b.id - a.id);
}
// 注意:不在此处预加载全部 SSTable 数据。
// 缓存容量有上限(cacheLimitBytes),全部预加载会突破内存预算。
// 读取路径由 prefetchRange / prefetchKeys 在查询前异步加载兜底。
// id 序列由 sstableStore.allocateId() 按命名空间独立恢复。
this.initialized = true;
}
// =======================================================================
// 写入
// =======================================================================
put(key, value) {
// 写背压:level 0 SSTable 过多时排队 compaction 缓解压力
if (this.levels[0].length >= 8) {
this.enqueueCompact(0);
}
this.memtable.put(key, value);
this.operationCount++;
if (this.memtable.shouldFlush()) {
this.freezeMemtable();
}
}
delete(key) {
if (this.levels[0].length >= 8) {
this.enqueueCompact(0);
}
this.memtable.put(key, { __tombstone: true });
this.operationCount++;
if (this.memtable.shouldFlush()) {
this.freezeMemtable();
}
}
/** 获取估算内存使用(字节) */
getEstimatedMemory() {
let mem = this.memtable.getEstimatedSize();
for (const frozen of this.frozenMemtables)
mem += frozen.getEstimatedSize();
mem += this.cacheSize;
return mem;
}
/**
* 冻结当前 MemTable 为 immutable,并在串行链上排队异步刷盘。
* 冻结的 MemTable 通过闭包捕获,避免链中前一个 flush 错误处理后续冻结的表。
* 所有 pending frozen 记录在 frozenMemtables 中,flush 完成前读取路径仍可访问。
*
* v0.4.2-fix: 链上任务失败时吞错恢复链(否则 flushChain 永久 rejected
* 后续所有 flush/compaction 挂起,写路径卡死)。
*/
freezeMemtable() {
if (this.immutableMemtable) {
const frozen = this.immutableMemtable;
this.flushChain = this.enqueueOnChain(() => this.flushImmutableAsync(frozen));
}
this.immutableMemtable = this.memtable;
this.frozenMemtables.push(this.immutableMemtable);
// v0.4.2-fix: 新 memtable 用配置阈值(此前传旧表已用大小 → 阈值逐次衰减 → 频繁小文件 flush)
this.memtable = new MemTable(this.memtableSizeThreshold);
}
/** v0.4.2-fix: 在串行链上排队任务;任务失败吞错并记录,保证链不被单次失败卡死 */
enqueueOnChain(task) {
return this.flushChain
.then(task)
.catch((error) => {
// v0.4.3-fix: 记录失败(flush()/close() 时报告),不再完全静默吞错
this.lastBackgroundError = error;
// eslint-disable-next-line no-console
console.warn('[AriaEngine LSM] background flush/compaction failed:', error);
// catch 返回 undefined → 链恢复为 resolved,后续任务继续
});
}
/**
* v0.4.3-fix: 排空后台链 — 循环等待 flushChain 直到稳定。
* 任务完成时可能级联调度新任务(compaction 多级触发),单次 await 等不到。
* close/flush/clear 必须等待全部后台任务完成后才能安全关闭底层存储。
*/
async drainChain() {
while (true) {
const chain = this.flushChain;
await chain;
if (this.flushChain === chain)
return;
}
}
/** 将指定 Immutable MemTable 刷盘为 SSTableid 由 store 按命名空间分配) */
async flushImmutableAsync(frozen) {
const entries = frozen.getAllEntries();
if (entries.length === 0) {
if (this.immutableMemtable === frozen)
this.immutableMemtable = null;
return;
}
const id = await this.sstableStore.allocateId();
const builder = new SSTableBuilder(this.blockSize);
for (const [key, value] of entries) {
builder.add(key, value);
}
const { sstableData, indexEntries } = builder.build();
const meta = {
id,
level: 0,
minKey: entries[0][0],
maxKey: entries[entries.length - 1][0],
blockCount: indexEntries.length,
totalSize: sstableData.byteLength,
bloomData: null,
};
// 缓存
this.cacheSSTable(id, sstableData);
this.trimCache();
// 持久化:先存数据,再存元数据(串行链保证顺序与 id 一致)
await this.sstableStore.save(id, sstableData);
await this.sstableStore.saveMeta(meta);
if (this.immutableMemtable === frozen)
this.immutableMemtable = null;
this.levels[0].unshift(meta);
// 数据已落盘(levels 可见),从 pending frozen 移除
this.frozenMemtables = this.frozenMemtables.filter((f) => f !== frozen);
// 异步触发 compaction(不阻塞当前写入)
if (this.levels[0].length >= 4 && !this.compacting) {
this.scheduleCompact(0);
}
}
/**
* 异步调度 compaction。
* v0.4.3-fix: 去掉 setTimeout 分片 — 此前未触发的定时器在 close 后执行,
* 用已关闭的 backend 写存储(错误被吞),或 close 后 reopen 时旧闭包引用新 backend 交叉污染。
* 现在直接挂在 flushChain 上:串行执行、close 的 drainChain 能等到全部完成。
*/
scheduleCompact(level) {
if (level >= MAX_LSM_LEVELS - 1 || this.compacting)
return;
this.compacting = true;
this.flushChain = this.enqueueOnChain(() => this.compactLevelAsync(level).finally(() => {
this.compacting = false;
// 连续触发:如果 compaction 后仍然超标,继续调度
if (this.levels[level].length >= 4) {
this.scheduleCompact(level);
}
// 检查下一级是否需要 compaction
if (level + 1 < MAX_LSM_LEVELS - 1 && this.levels[level + 1].length >= 4) {
this.scheduleCompact(level + 1);
}
}));
}
/** 背压场景下排队 compaction(写入路径调用) */
enqueueCompact(level) {
if (level >= MAX_LSM_LEVELS - 1 || this.compacting)
return;
this.compacting = true;
this.flushChain = this.flushChain.then(async () => {
try {
await this.compactLevelAsync(level);
}
finally {
this.compacting = false;
}
}).catch((error) => {
this.compacting = false;
// v0.4.3-fix: 记录失败(flush()/close() 时报告)
this.lastBackgroundError = error;
// eslint-disable-next-line no-console
console.warn('[AriaEngine LSM] background compaction failed:', error);
});
}
// =======================================================================
// 读取
// =======================================================================
/**
* 预加载指定 key 范围内可能命中的所有 SSTable 到缓存。
* 在同步扫描/查找之前调用,保证 loadSSTableReader 不会因缓存未命中而返回 null。
* v0.4.3-fix: 等待链稳定(drainChain)— 后台 flush/compaction 在链上动态增长,
* 单次 await 后 compaction 仍可能合并 levels,导致扫描时新 meta 缓存未命中而跳块丢数据。
*/
async prefetchRange(startKey, endKey) {
await this.drainChain();
this.trimCache();
const toLoad = [];
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
for (const meta of this.levels[level]) {
if (endKey < meta.minKey || startKey > meta.maxKey)
continue;
if (!this.sstableCache.has(meta.id))
toLoad.push(meta.id);
}
}
for (const id of toLoad) {
const meta = this.findMetaById(id);
await this.preloadSSTable(id, meta);
}
}
/** 预加载包含指定 key 的所有 SSTable 到缓存 */
async prefetchKeys(keys) {
if (keys.length === 0)
return;
// v0.4.3-fix: 等待链稳定(同 prefetchRange,防 compaction 竞态丢数据)
await this.drainChain();
this.trimCache();
const toLoad = new Set();
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
for (const meta of this.levels[level]) {
if (this.sstableCache.has(meta.id))
continue;
for (const key of keys) {
if (key >= meta.minKey && key <= meta.maxKey) {
toLoad.add(meta.id);
break;
}
}
}
}
for (const id of toLoad) {
const meta = this.findMetaById(id);
await this.preloadSSTable(id, meta);
}
}
/**
* v0.6.2: 批量预加载多个前缀范围可能命中的所有 SSTable(一次 drainChain)。
* 唯一性检查等"每行一个窄前缀范围"场景用:meta 遍历只做一次,
* 避免逐行调用 prefetchRange 时每行 drainChain 的性能悬崖。
*/
async prefetchPrefixRanges(ranges) {
if (ranges.length === 0)
return;
// v0.6.2: 去重(大批量插入时唯一列值可能重复),减少 meta 遍历开销
const seen = new Set();
const unique = [];
for (const r of ranges) {
const k = `${r[0]}\u0000${r[1]}`;
if (!seen.has(k)) {
seen.add(k);
unique.push(r);
}
}
if (unique.length === 0)
return;
await this.drainChain();
this.trimCache();
const toLoad = new Set();
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
for (const meta of this.levels[level]) {
if (this.sstableCache.has(meta.id))
continue;
for (const [startKey, endKey] of unique) {
if (endKey < meta.minKey || startKey > meta.maxKey)
continue;
toLoad.add(meta.id);
break;
}
}
}
for (const id of toLoad) {
const meta = this.findMetaById(id);
await this.preloadSSTable(id, meta);
}
}
/** 按 id 查找 SSTable 元数据(prefetch 预加载用) */
findMetaById(id) {
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
for (const meta of this.levels[level]) {
if (meta.id === id)
return meta;
}
}
return undefined;
}
get(key) {
// 1. 活跃 MemTable
let result = this.memtable.get(key);
if (result !== null)
return this.unwrapTombstone(result);
// 2. pending frozen memtables(从新到旧)
for (let i = this.frozenMemtables.length - 1; i >= 0; i--) {
result = this.frozenMemtables[i].get(key);
if (result !== null)
return this.unwrapTombstone(result);
}
// 3. SSTable(从 Level 0 到 Level N-1
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
for (const meta of this.levels[level]) {
if (key < meta.minKey || key > meta.maxKey)
continue;
const reader = this.loadSSTableReader(meta);
if (!reader)
continue;
const found = reader.get(key);
if (found !== null)
return this.unwrapTombstone(found);
}
}
return null;
}
rangeScan(startKey, endKey) {
const result = [];
this.rangeScanLazy(startKey, endKey, (k, v) => result.push([k, v]));
return result;
}
/** 惰性范围扫描:通过回调逐条返回,不一次性物化所有源 */
rangeScanLazy(startKey, endKey, callback) {
const mergeIter = new MergeIterator();
mergeIter.addSource(new ArrayEntrySource(this.memtable.rangeScan(startKey, endKey)));
// pending frozen memtables(从新到旧,新数据 sourceIndex 更小)
for (let i = this.frozenMemtables.length - 1; i >= 0; i--) {
mergeIter.addSource(new ArrayEntrySource(this.frozenMemtables[i].rangeScan(startKey, endKey)));
}
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
for (const meta of this.levels[level]) {
if (endKey < meta.minKey || startKey > meta.maxKey)
continue;
const reader = this.loadSSTableReader(meta);
if (!reader)
continue;
reader.rangeScan(startKey, endKey, (k, v) => {
mergeIter.addSource(new ArrayEntrySource([[k, v]]));
});
}
}
const merged = mergeIter.drain();
for (const [k, v] of merged) {
if (!v.__tombstone) {
callback(k, v);
}
}
}
getAllEntries() {
const result = new Map();
// 从最旧层级开始聚合;同层级内从最旧到最新遍历,
// 保证 result.set 覆盖时最终保留最新值
for (let level = MAX_LSM_LEVELS - 1; level >= 0; level--) {
for (let i = this.levels[level].length - 1; i >= 0; i--) {
const meta = this.levels[level][i];
const reader = this.loadSSTableReader(meta);
if (!reader)
continue;
reader.scanAll((k, v) => result.set(k, v));
}
}
// MemTable 覆盖(最新)
for (const [k, v] of this.memtable.getAllEntries()) {
result.set(k, v);
}
for (let i = this.frozenMemtables.length - 1; i >= 0; i--) {
for (const [k, v] of this.frozenMemtables[i].getAllEntries()) {
result.set(k, v);
}
}
return Array.from(result.entries()).filter(([, v]) => !v.__tombstone);
}
// =======================================================================
// Compaction
// =======================================================================
/** 执行 Compactionpublic,供 VACUUM 等外部调用;VACUUM 期望 2 个文件即可压缩) */
async compactLevel(level) {
await this.compactLevelAsync(level, 2);
}
/**
* 串行执行 Compaction。
* @param minFiles 触发压缩的文件数门槛(自动调度用 4,VACUUM 用 2)
*
* v0.4.2-fix: 读取从存储兜底(不依赖缓存)——此前仅从缓存读,
* 缓存未命中(LRU 驱逐/单文件超缓存上限)时跳过全部文件并从 levels 移除,
* 运行中数据全部不可见。
*/
async compactLevelAsync(level, minFiles = 4) {
if (level >= MAX_LSM_LEVELS - 1)
return;
if (this.levels[level].length < minFiles)
return;
const sstables = this.levels[level].splice(0, this.levels[level].length);
const mergeIter = new MergeIterator();
const loadedMetas = [];
for (const meta of sstables) {
// 优先缓存,未命中则从存储加载(残缺文件经校验清理,跳过)
let data = this.sstableCache.get(meta.id) ?? null;
if (!data) {
try {
data = await this.sstableStore.load(meta.id);
}
catch {
data = null;
}
}
if (!data || data.byteLength < 32) {
await this.dropInvalidSSTable(meta);
continue;
}
let reader;
try {
reader = new SSTableReader(data, meta);
if (!reader.verifyChecksum()) {
await this.dropInvalidSSTable(meta);
continue;
}
}
catch {
await this.dropInvalidSSTable(meta);
continue;
}
const entries = [];
reader.scanAll((k, v) => entries.push([k, v]));
mergeIter.addSource(new ArrayEntrySource(entries));
loadedMetas.push(meta);
}
const merged = mergeIter.drain();
if (merged.length === 0) {
// 没有有效数据(全部损坏):把有效 meta 放回 levels
// 避免文件从读取路径消失(数据仍在磁盘,重启可恢复)
for (const meta of loadedMetas) {
this.levels[level].push(meta);
}
this.levels[level].sort((a, b) => b.id - a.id);
return;
}
const id = await this.sstableStore.allocateId();
const builder = new SSTableBuilder(this.blockSize);
for (const [key, value] of merged) {
builder.add(key, value);
}
const { sstableData, indexEntries } = builder.build();
const meta = {
id,
level: level + 1,
minKey: merged[0][0],
maxKey: merged[merged.length - 1][0],
blockCount: indexEntries.length,
totalSize: sstableData.byteLength,
bloomData: null,
};
this.cacheSSTable(id, sstableData);
this.trimCache();
await this.sstableStore.save(id, sstableData);
await this.sstableStore.saveMeta(meta);
this.levels[level + 1].unshift(meta);
// 删除旧 SSTable
for (const old of sstables) {
this.sstableCache.delete(old.id);
await this.sstableStore.delete(old.id);
await this.sstableStore.deleteMeta(old.id);
}
}
/** 等待所有排队的 flush/compaction 完成,并将剩余数据刷盘 */
async flush() {
// v0.4.3-fix: 报告后台失败(消费一次,不永久吞错)
if (this.lastBackgroundError !== null) {
const error = this.lastBackgroundError;
this.lastBackgroundError = null;
throw new DatabaseError('AriaEngine background flush/compaction failed (data may be inconsistent)', 'ARIA_BACKGROUND_ERROR', error);
}
// v0.4.3-fix: 循环等待级联任务(flush 完成可能触发新的 compaction
await this.drainChain();
// v0.6.1-fix(P0): 剩余数据 flush 必须挂链串行 —— 此前直接 await 执行,
// 与链上 compaction 并发写 SSTable metacompaction 产物 saveMeta 后,
// memtable flush 的 saveMeta 读到中间态列表(含 compaction 产物)→ 覆盖产物
// 引用 → compaction 产物变孤儿 → 索引/主表数据静默丢失(优雅关闭后重开丢 75%)。
// 挂链后按序执行(memtable flush 在 compaction 之后),meta 无竞态。
//
// v0.6.3-fix: ① 入链后立即置空 immutable —— 此前 freezeMemtable 会把同一张
// immutable 再次入链 → 重复 SSTable2 行 flush 出 3 个文件);② frozenMemtables
// 保持到链排空后再清理 —— 此前入链后立即清空,链执行期间并发读看不到冻结数据
// (短暂数据不可见窗口)。
if (this.immutableMemtable) {
const frozen = this.immutableMemtable;
this.flushChain = this.enqueueOnChain(() => this.flushImmutableAsync(frozen));
this.immutableMemtable = null;
}
if (this.memtable.getEntryCount() > 0) {
this.freezeMemtable();
const frozen = this.immutableMemtable;
if (frozen) {
this.flushChain = this.enqueueOnChain(() => this.flushImmutableAsync(frozen));
this.immutableMemtable = null;
}
}
// 刷盘完成后级联调度可能触发 compaction → 排空到稳定
await this.drainChain();
// 清理空残留冻结表(flushImmutableAsync 完成时已自行移除已落盘的)
this.frozenMemtables = this.frozenMemtables.filter((f) => f.getEntryCount() > 0);
}
async clear() {
// v0.4.3-fix: 清空前排空后台任务(避免 compaction 在清空后写回残留 meta/数据)
await this.drainChain();
this.memtable.clear();
this.immutableMemtable = null;
this.frozenMemtables = [];
for (const level of this.levels) {
for (const meta of level) {
this.sstableCache.delete(meta.id);
this.sstableStore.delete(meta.id).catch(() => { });
this.sstableStore.deleteMeta(meta.id).catch(() => { });
}
}
this.levels = [];
for (let i = 0; i < MAX_LSM_LEVELS; i++) {
this.levels.push([]);
}
this.sstableCache.clear();
this.cacheSize = 0;
}
getStats() {
return {
memtableSize: this.memtable.getEntryCount(),
sstableCount: this.levels.reduce((sum, l) => sum + l.length, 0),
levelCounts: this.levels.map((l) => l.length),
};
}
// =======================================================================
// 内部
// =======================================================================
/**
* v0.4.2-fix: 重新校验全部已加载 SSTable,移除损坏项(repair 自愈用)。
* @returns 移除的损坏 SSTable 数量
*/
async validateAll() {
let removed = 0;
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
const valid = [];
for (const meta of this.levels[level]) {
if (await this.validateSSTable(meta)) {
valid.push(meta);
}
else {
this.sstableCache.delete(meta.id);
removed++;
}
}
this.levels[level] = valid;
}
return removed;
}
/**
* v0.4.2-fix: 校验单个 SSTable 的完整性。
* - 文件不存在 → 清理 meta,返回 false
* - 文件过小/魔数错误/索引越界(残缺写入产物)→ 清理 meta,返回 false
* - v0.4.5: 整文件 CRC-32 校验失败(数据腐坏)→ 清理 meta,返回 false
* 校验通过的数据不缓存(保持内存预算),读路径按需预加载。
*/
async validateSSTable(meta) {
try {
const data = await this.sstableStore.load(meta.id);
if (!data) {
this.dropInvalidSSTable(meta);
return false;
}
if (data.byteLength < 32) {
this.dropInvalidSSTable(meta);
return false;
}
try {
const reader = new SSTableReader(data, meta);
if (!reader.verifyChecksum()) {
this.dropInvalidSSTable(meta);
return false;
}
}
catch {
this.dropInvalidSSTable(meta);
return false;
}
return true;
}
catch {
this.dropInvalidSSTable(meta);
return false;
}
}
/** 清理无效 SSTable 的 meta 与文件(打开自愈路径) */
async dropInvalidSSTable(meta) {
// eslint-disable-next-line no-console
console.warn(`[AriaEngine LSM] Skipping corrupted SSTable id=${meta.id} (level=${meta.level})`);
// v0.6.0-fix: 先删数据文件再删 meta — 页面化存储的 delete 依赖 meta.pageIds
// 定位页面文件;先删 meta 会丢失 pageIds 导致孤儿页面残留
try {
await this.sstableStore.delete(meta.id);
}
catch { /* 清理失败不阻塞打开 */ }
try {
await this.sstableStore.deleteMeta(meta.id);
}
catch { /* 清理失败不阻塞打开 */ }
}
unwrapTombstone(value) {
if (!value)
return null;
if (value.__tombstone)
return null;
return value;
}
/** 尝试从缓存或存储加载 SSTable,返回 Reader */
loadSSTableReader(meta) {
// 先检查缓存
const data = this.sstableCache.get(meta.id);
if (!data) {
// 缓存未命中:正常路径应在查询前通过 prefetchRange/prefetchKeys 预加载。
// 这里仅当缓存中有缺失且无兜底时返回 null(调用方跳过)。
return null;
}
// 刷新 LRU 顺序(近似:删除后重新插入使其成为最近使用)
this.sstableCache.delete(meta.id);
this.sstableCache.set(meta.id, data);
try {
return new SSTableReader(data, meta);
}
catch {
return null;
}
}
/** 预加载 SSTable 到缓存(受 cacheLimitBytes 上限约束) */
async preloadSSTable(id, meta) {
if (this.sstableCache.has(id))
return;
const data = await this.sstableStore.load(id);
if (!data)
return;
// v0.4.5: 运行期加载同样校验整文件 CRC-32,损坏文件不缓存并清理(自愈)
if (meta) {
try {
const reader = new SSTableReader(data, meta);
if (!reader.verifyChecksum()) {
await this.dropInvalidSSTable(meta);
return;
}
}
catch {
await this.dropInvalidSSTable(meta);
return;
}
}
this.cacheSSTable(id, data);
}
/** 写入缓存。注意:不在加载时立即驱逐,避免破坏正在进行的同步扫描 */
cacheSSTable(id, data) {
// 已存在则先移除(保持"最近使用"语义)
if (this.sstableCache.has(id)) {
this.cacheSize -= this.sstableCache.get(id).byteLength;
this.sstableCache.delete(id);
}
this.sstableCache.set(id, data);
this.cacheSize += data.byteLength;
}
/**
* LRU 裁剪:从最早插入的条目开始驱逐,直到缓存总字节数不超过 cacheLimitBytes。
* 仅在查询/写入开始前调用,保证本次查询所需数据在同步扫描期间全部存活。
* 查询结束后由引擎调用一次,回收查询期间的临时超限。
*/
trimCache() {
while (this.cacheSize > this.cacheLimitBytes && this.sstableCache.size > 0) {
const eldestId = this.sstableCache.keys().next().value;
const evicted = this.sstableCache.get(eldestId);
this.cacheSize -= evicted.byteLength;
this.sstableCache.delete(eldestId);
}
}
}
/**
* AriaEngine WAL — Write-Ahead Log
* @module engine/aria/wal/log
*
* 崩溃恢复前的写操作持久化日志。
*
* WAL 文件格式:
* ┌──────────┬──────────────┬──────────┐
* │ Record 1│ Record 2 │ ... │
* │ 4B LSN │ │ │
* │ 1B type │ │ │
* │ 4B txnId│ │ │
* │ 2B tblLen│ │ │
* │ N table│ │ │
* │ 2B keyLen│ │ │
* │ N key │ │ │
* │ 4B jsonLen│ │ │
* │ N json │ │ │
* │ 4B CRC │ │ │
* └──────────┴──────────────┴──────────┘
*/
// ---------------------------------------------------------------------------
// WAL
// ---------------------------------------------------------------------------
class WAL {
constructor(store, enabled = true, syncMode = 'batch') {
this.lsn = 0;
this.buffer = [];
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数(full/batch/none 通用) */
this.bufferedBytes = 0;
this.store = store;
this.enabled = enabled;
this.syncMode = syncMode;
}
// =======================================================================
// 写入
// =======================================================================
/** 追加一条 WAL 记录(full 模式同步等待写入完成) */
async append(record) {
if (!this.enabled)
return;
this.lsn++;
const fullRecord = {
...record,
lsn: this.lsn,
checksum: 0, // 稍后计算
};
const bytes = this.encodeRecord(fullRecord);
if (this.syncMode === 'full') {
// v0.6.3-fix: 写入失败必须抛给调用方 —— 此前仅 console.warn 吞错:
// 内存已提交而 WAL 缺失,崩溃即丢且调用方无感知。
await this.store.append(bytes);
this.bufferedBytes += bytes.byteLength;
}
else if (this.syncMode === 'batch') {
this.buffer.push(bytes);
this.bufferedBytes += bytes.byteLength;
}
// 'none' mode: 不写 WAL
}
/** 批量追加多条 WAL 记录(组提交:合并为一次底层写入,v0.3.1) */
async appendBatch(records) {
if (!this.enabled || records.length === 0)
return;
const chunks = [];
for (const record of records) {
this.lsn++;
chunks.push(this.encodeRecord({ ...record, lsn: this.lsn, checksum: 0 }));
}
const combined = this.mergeChunks(chunks);
if (this.syncMode === 'full') {
// v0.6.3-fix: 同 append —— 批量写入失败抛给调用方(不再吞错)
await this.store.append(combined);
this.bufferedBytes += combined.byteLength;
}
else if (this.syncMode === 'batch') {
this.buffer.push(combined);
this.bufferedBytes += combined.byteLength;
}
// 'none' mode: 不写 WAL
}
/** 批量刷新缓冲的 WAL 记录 */
async flush() {
if (!this.enabled || this.buffer.length === 0)
return;
const combined = this.mergeChunks(this.buffer);
await this.store.append(combined);
this.buffer = [];
}
/** 合并多个字节块为一个连续缓冲区 */
mergeChunks(chunks) {
if (chunks.length === 1)
return chunks[0];
const totalLen = chunks.reduce((sum, b) => sum + b.byteLength, 0);
const combined = new Uint8Array(totalLen);
let offset = 0;
for (const buf of chunks) {
combined.set(buf, offset);
offset += buf.byteLength;
}
return combined;
}
// =======================================================================
// 恢复
// =======================================================================
/** 从 WAL 恢复未提交的事务数据 */
async recover(applyRecord) {
if (!this.enabled)
return 0;
const exists = await this.store.exists();
if (!exists)
return 0;
const data = await this.store.readAll();
if (data.byteLength === 0)
return 0;
const records = this.decodeAllRecords(data);
for (const record of records) {
applyRecord(record);
}
this.lsn = records.length > 0 ? records[records.length - 1].lsn : 0;
return records.length;
}
// =======================================================================
// Checkpoint
// =======================================================================
/** Checkpoint 后清空 WAL */
async checkpoint() {
if (!this.enabled)
return;
await this.flush();
await this.store.truncate();
this.lsn = 0;
this.bufferedBytes = 0;
}
// =======================================================================
// 统计
// =======================================================================
getBufferedCount() {
return this.buffer.length;
}
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数(full/batch/none 通用) */
getBufferedBytes() {
return this.bufferedBytes;
}
// -----------------------------------------------------------------------
// 编解码
// -----------------------------------------------------------------------
/** 旧版弱滚动校验(v0.4.4 及更早写入的 WAL 记录使用,双算法探测兼容) */
legacyChecksum(data) {
let crc = 0;
for (let i = 0; i < data.length; i++) {
crc = ((crc << 5) - crc + data[i]) | 0;
}
return crc >>> 0;
}
encodeRecord(record) {
const encoder = new TextEncoder();
const tableBytes = encoder.encode(record.tableName);
const keyBytes = encoder.encode(record.key);
const jsonStr = record.data ? JSON.stringify(record.data) : '';
const jsonBytes = encoder.encode(jsonStr);
const size = 4 + // LSN
1 + // type
4 + // txnId
2 + tableBytes.length + // table
2 + keyBytes.length + // key
4 + jsonBytes.length + // json
4; // CRC
const buf = new ArrayBuffer(size);
const view = new DataView(buf);
let offset = 0;
view.setUint32(offset, record.lsn, false);
offset += 4;
view.setUint8(offset, record.type);
offset += 1;
view.setUint32(offset, record.txnId, false);
offset += 4;
view.setUint16(offset, tableBytes.length, false);
offset += 2;
new Uint8Array(buf).set(tableBytes, offset);
offset += tableBytes.length;
view.setUint16(offset, keyBytes.length, false);
offset += 2;
new Uint8Array(buf).set(keyBytes, offset);
offset += keyBytes.length;
view.setUint32(offset, jsonBytes.length, false);
offset += 4;
new Uint8Array(buf).set(jsonBytes, offset);
offset += jsonBytes.length;
// v0.4.5: 标准 CRC-32 校验(此前为弱滚动校验,误检率更高)
const u8 = new Uint8Array(buf, 0, offset);
const crc = crc32(u8);
view.setUint32(offset, crc, false);
return new Uint8Array(buf);
}
decodeAllRecords(data) {
const records = [];
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
let offset = 0;
while (offset + 15 <= data.byteLength) {
try {
const recordStart = offset;
const lsn = view.getUint32(offset, false);
offset += 4;
const type = view.getUint8(offset);
offset += 1;
const txnId = view.getUint32(offset, false);
offset += 4;
const tableLen = view.getUint16(offset, false);
offset += 2;
if (offset + tableLen > data.byteLength)
break;
const tableName = new TextDecoder().decode(data.slice(offset, offset + tableLen));
offset += tableLen;
const keyLen = view.getUint16(offset, false);
offset += 2;
if (offset + keyLen > data.byteLength)
break;
const key = new TextDecoder().decode(data.slice(offset, offset + keyLen));
offset += keyLen;
const jsonLen = view.getUint32(offset, false);
offset += 4;
if (offset + jsonLen > data.byteLength)
break;
let recordData;
if (jsonLen > 0) {
const json = new TextDecoder().decode(data.slice(offset, offset + jsonLen));
try {
recordData = JSON.parse(json);
}
catch { /* ok */ }
}
offset += jsonLen;
// 验证 CRC:先标准 CRC-32,失败再尝试旧版弱滚动校验(兼容旧库 WAL 记录)
const storedCrc = view.getUint32(offset, false);
offset += 4;
const recordBytes = data.slice(recordStart, offset - 4);
const computedNew = crc32(recordBytes);
const computedLegacy = this.legacyChecksum(recordBytes);
if ((computedNew >>> 0) !== storedCrc && (computedLegacy >>> 0) !== storedCrc) {
// CRC 不匹配,跳过此损坏记录
// eslint-disable-next-line no-console
console.warn(`[AriaEngine WAL] CRC mismatch at record LSN=${lsn}, skipping`);
continue;
}
records.push({
lsn,
type,
txnId,
tableName,
key,
data: recordData,
checksum: storedCrc,
});
}
catch {
break;
}
}
return records;
}
}
/**
* AriaEngine Segmented WAL Store — 分片式 WAL 持久化存储
* @module engine/aria/wal/segmented_store
*
* v0.4.5: 取代"每条记录一个 key"的旧结构:
* - 分片文件 `__wal_%06d.bin`,达到阈值(默认 4MB)切新分片 → 文件数量可控
* - append 追加写当前分片(后端支持真追加则 O(chunk),否则回退 read+write
* - WAL 序号(LSN)内嵌于记录字节流,无需独立 count 键 → append 单文件原子写
* - readAll 检测分片序号空洞:序号不连续 → 丢弃空洞之后的分片(保守截断,
* 空洞仅可能来自 truncate 部分完成——此时数据已落盘,丢弃无害)
* - 兼容旧格式 `__wal_N`(每条记录一个键)+ `__wal_count`:读取时迁移重放,
* checkpoint 时一并清空
*/
/** 分片文件名:__wal_%06d.bin */
const WAL_SEGMENT_PREFIX = '__wal_';
const SEGMENT_REGEX = /^__wal_(\d{6})\.bin$/;
const LEGACY_RECORD_REGEX = /^__wal_(\d+)$/;
const LEGACY_COUNT_KEY = '__wal_count';
/** 默认分片阈值 */
const DEFAULT_WAL_SEGMENT_SIZE = 4 * 1024 * 1024;
// ---------------------------------------------------------------------------
// SegmentedWALStore
// ---------------------------------------------------------------------------
class SegmentedWALStore {
constructor(backend, segmentSize = DEFAULT_WAL_SEGMENT_SIZE) {
this.backend = backend;
/** 当前分片序号(append 定位) */
this.currentSegment = 0;
/** 当前分片字节数(内存跟踪,append 切分片判断) */
this.currentSize = 0;
this.segmentSize = segmentSize;
}
/** 分片 key 生成 */
segmentKey(seq) {
return `${WAL_SEGMENT_PREFIX}${String(seq).padStart(6, '0')}.bin`;
}
async append(data) {
if (data.byteLength === 0)
return;
if (this.currentSize + data.byteLength > this.segmentSize) {
// 当前分片放不下 → 切新分片
this.currentSegment++;
this.currentSize = 0;
}
const key = this.segmentKey(this.currentSegment);
const copy = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
if (typeof this.backend.append === 'function') {
await this.backend.append(key, copy);
}
else {
// 回退:读旧 + 拼接 + 写(单文件原子写)
const existing = await this.backend.read(key);
if (existing) {
const combined = new ArrayBuffer(existing.byteLength + copy.byteLength);
new Uint8Array(combined).set(new Uint8Array(existing), 0);
new Uint8Array(combined).set(new Uint8Array(copy), existing.byteLength);
await this.backend.write(key, combined);
}
else {
await this.backend.write(key, copy);
}
}
this.currentSize += data.byteLength;
}
async readAll() {
const keys = await this.backend.listKeys();
// ---- 新格式分片 ----
const segments = keys
.filter((k) => SEGMENT_REGEX.test(k))
.map((k) => ({ seq: Number(k.match(SEGMENT_REGEX)[1]), key: k }))
.sort((a, b) => a.seq - b.seq);
// 空洞检测:分片序号必须从 0 严格连续,空洞后的分片整体丢弃
let keepCount = 0;
for (let i = 0; i < segments.length; i++) {
if (segments[i].seq !== i)
break;
keepCount = i + 1;
}
const validSegments = segments.slice(0, keepCount);
// ---- 旧格式兼容:__wal_N 单记录键(迁移前数据) ----
const legacyKeys = keys
.filter((k) => LEGACY_RECORD_REGEX.test(k))
.map((k) => ({ seq: Number(k.match(LEGACY_RECORD_REGEX)[1]), key: k }))
.sort((a, b) => a.seq - b.seq);
const parts = [];
// 旧格式在前(它们是最早的记录)
for (const { key } of legacyKeys) {
const raw = await this.backend.read(key);
if (raw)
parts.push(new Uint8Array(raw));
}
// 新格式分片在后
for (const { key } of validSegments) {
const raw = await this.backend.read(key);
if (raw)
parts.push(new Uint8Array(raw));
}
// 同步当前分片状态(追加定位)
if (validSegments.length > 0) {
this.currentSegment = validSegments[validSegments.length - 1].seq;
const lastRaw = await this.backend.read(validSegments[validSegments.length - 1].key);
this.currentSize = lastRaw ? lastRaw.byteLength : 0;
// 旧格式键存在时(迁移中),下一条记录另起分片,避免与旧键序号冲突
if (legacyKeys.length > 0) {
this.currentSegment++;
this.currentSize = 0;
}
}
else if (legacyKeys.length > 0) {
// 仅有旧格式:迁移中,新写入从分片 0 开始(checkpoint 会清空旧键)
this.currentSegment = 0;
this.currentSize = 0;
}
const total = parts.reduce((s, c) => s + c.byteLength, 0);
const combined = new Uint8Array(total);
let off = 0;
for (const c of parts) {
combined.set(c, off);
off += c.byteLength;
}
return combined;
}
async truncate() {
const keys = await this.backend.listKeys();
const walKeys = keys.filter((k) => k.startsWith(WAL_SEGMENT_PREFIX) || k === LEGACY_COUNT_KEY);
if (walKeys.length > 0) {
await this.backend.deleteMany(walKeys);
}
this.currentSegment = 0;
this.currentSize = 0;
}
async exists() {
const keys = await this.backend.listKeys();
return keys.some((k) => k.startsWith(WAL_SEGMENT_PREFIX) || k === LEGACY_COUNT_KEY);
}
}
/**
* AriaEngine Database Lock — 多标签页独占锁(Web Locks API
* @module engine/aria/locks
*
* v0.4.5: OPFS 等无事务后端缺乏多标签页并发协调,多个标签页同时打开同一库
* 会导致写竞态与数据损坏。用 Web Locks APIChrome 69+ / Firefox 96+ / Safari 15.4+
* 获取库级排他锁:
* - ifAvailable 模式:锁被其他标签页持有 → 立即抛 ARIA_LOCKED(不排队挂起)
* - 持锁期间回调挂起,close() 时释放
* - 浏览器不支持 navigator.locks → 返回 false(如实降级:无并发保护,文档注明)
*/
/** Web Locks 锁名(库级排他) */
function lockName(dbName) {
return `metona-sqlark:${dbName}`;
}
class DatabaseLock {
constructor() {
this.acquired = false;
this.supported = false;
this.releaseResolve = null;
this.releasePromise = null;
}
/**
* 尝试获取独占锁。
* @returns true = 已持锁;false = 环境不支持 Web Locks(无并发保护,调用方可警告)
* @throws ARIA_LOCKED 锁被其他标签页持有
*/
async acquire(dbName) {
const nav = globalThis.navigator;
const lockManager = nav?.locks;
if (!lockManager || typeof lockManager.request !== 'function') {
this.supported = false;
return false;
}
this.supported = true;
await new Promise((resolve, reject) => {
// 注意:必须直接调用(不能解构 request —— LockManager 方法依赖 this 绑定)
const request = lockManager.request.bind(lockManager);
request(lockName(dbName), { ifAvailable: true, mode: 'exclusive' }, async (lock) => {
if (!lock) {
reject(new DatabaseError(`Database "${dbName}" is already open in another tab (locked)`, 'ARIA_LOCKED'));
return;
}
this.acquired = true;
const releasePromise = new Promise((res) => { this.releaseResolve = res; });
this.releasePromise = releasePromise;
// 锁已获取:acquire 返回(open 流程继续)
resolve();
// 回调挂起:保持锁直到 release() 触发
await releasePromise;
});
});
return true;
}
/** 释放锁(等待回调真正结束,保证锁已归还) */
async release() {
if (!this.acquired)
return;
if (this.releaseResolve) {
const res = this.releaseResolve;
const p = this.releasePromise;
this.releaseResolve = null;
this.releasePromise = null;
res();
try {
await p;
}
catch { /* 释放过程异常不阻塞 */ }
}
this.acquired = false;
}
/** 是否已持锁 */
isAcquired() {
return this.acquired;
}
/** 环境是否支持 Web Locks */
isSupported() {
return this.supported;
}
}
/**
* AriaEngine Checkpoint — 检查点机制
* @module engine/aria/wal/checkpoint
*/
// ---------------------------------------------------------------------------
// CheckpointManager
// ---------------------------------------------------------------------------
class CheckpointManager {
constructor(lsm, wal, flushable = null, interval = 1000, walSizeThreshold = 16 * 1024 * 1024) {
this.opCount = 0;
this.lsm = lsm;
this.wal = wal;
this.flushable = flushable;
this.interval = interval;
this.walSizeThreshold = walSizeThreshold;
}
async tick() {
this.opCount++;
// 检查操作计数或 WAL 大小是否超阈值
if (this.opCount >= this.interval || this.getWALEstimatedSize() >= this.walSizeThreshold) {
await this.checkpoint();
}
}
/** 估算 WAL 大小(优先真实字节数,回退到缓冲计数估算) */
getWALEstimatedSize() {
const wal = this.wal;
if (typeof wal.getBufferedBytes === 'function') {
const bytes = wal.getBufferedBytes();
if (bytes > 0)
return bytes;
}
const count = typeof wal.getBufferedCount === 'function' ? wal.getBufferedCount() : 0;
return count * 200;
}
async checkpoint() {
await this.lsm.flush();
if (this.flushable) {
await this.flushable.flushAll();
}
await this.wal.checkpoint();
this.opCount = 0;
}
}
/**
* AriaEngine Storage Backend — 存储后端抽象层
* @module engine/aria/store/backend
*
* 封装底层浏览器存储 APIIndexedDB / OPFS / Memory 回退),
* 供 Buffer Pool 的 PageIO 和 WAL 的 WALStore 使用。
*/
// =======================================================================
// Memory Backend(回退 / 测试用)
// =======================================================================
class MemoryBackend {
constructor() {
this.store = new Map();
this.opened = false;
}
async open(_name) {
this.opened = true;
}
async close() {
this.store.clear();
this.opened = false;
}
isOpen() {
return this.opened;
}
async read(key) {
return this.store.get(key) ?? null;
}
async write(key, data) {
this.store.set(key, data);
}
async writeMany(entries) {
for (const [key, data] of Object.entries(entries)) {
this.store.set(key, data);
}
}
async delete(key) {
this.store.delete(key);
}
async deleteMany(keys) {
for (const key of keys) {
this.store.delete(key);
}
}
async listKeys() {
return Array.from(this.store.keys());
}
async exists(key) {
return this.store.has(key);
}
async clear() {
this.store.clear();
}
}
/**
* KVStoreBackend — 基于自研 KVStore 的 AriaEngine 存储后端
* @module engine/aria/store/kvstore_backend
*
* v0.6.1: AriaEngine 可选后端(storageBackend: 'kv'),完全跑在自研 KVStore 上:
* - write/read/delete/listKeys/exists/clear → KVStore 直接映射
* - append → KVStore.appendValue(日志 APPEND 类型,O(chunk) 高效,aria WAL 分片用)
* - writeMany/deleteMany → KVStore.putMany/deleteMany(单日志记录真原子,
* 此前 OPFS 逐文件写靠空洞检测兜底,KV 后端原生原子)
* - 崩溃恢复:KVStore 快照+日志恢复 → aria 打开重放自己的 WAL(双层恢复)
*
* 介质:浏览器 OPFSKVStore 默认)或 Node SharedMemory——aria 不再依赖浏览器 OPFS API。
*/
class KVStoreBackend {
constructor(medium, checkpointThreshold) {
this.kv = new KVStore(medium, checkpointThreshold);
}
/** 底层 KVStore(测试/诊断用) */
getKV() {
return this.kv;
}
async open(name) {
await this.kv.open(name);
}
async close() {
await this.kv.close();
}
isOpen() {
return this.kv.isOpen();
}
async read(key) {
return this.kv.get(key);
}
async write(key, data) {
await this.kv.put(key, data);
}
/** 追加写入(KVStore APPEND 日志,O(chunk) */
async append(key, data) {
await this.kv.appendValue(key, data);
}
/** 多 key 原子写入(单日志记录) */
async writeMany(entries) {
await this.kv.putMany(entries);
}
async delete(key) {
await this.kv.delete(key);
}
/** 多 key 原子删除(单日志记录) */
async deleteMany(keys) {
await this.kv.deleteMany(keys);
}
async listKeys() {
return this.kv.listKeys();
}
async exists(key) {
return this.kv.exists(key);
}
async clear() {
await this.kv.clear();
}
}
/**
* AriaEngine Crypto — 页面级 AES-GCM 加密
* @module engine/aria/crypto
*
* v0.2.5: 改为实例化 CryptoManager,避免多实例共享全局状态。
* 保留全局函数兼容旧代码(委托给全局单例)。
*/
const ALGO = 'AES-GCM';
const IV_LENGTH$1 = 12;
/**
* CryptoManager — 实例级加密管理器
* 每个 AriaEngine 实例可拥有独立的加密配置。
*/
class CryptoManager {
constructor() {
this.cryptoKey = null;
this._enabled = false;
}
get enabled() { return this._enabled; }
async init(password, salt) {
const enc = new TextEncoder();
const keyMaterial = await crypto.subtle.importKey('raw', enc.encode(password), 'PBKDF2', false, ['deriveKey']);
const actualSalt = salt || crypto.getRandomValues(new Uint8Array(16));
this.cryptoKey = await crypto.subtle.deriveKey({ name: 'PBKDF2', salt: actualSalt, iterations: 100000, hash: 'SHA-256' }, keyMaterial, { name: ALGO, length: 256 }, false, ['encrypt', 'decrypt']);
this._enabled = true;
return actualSalt;
}
async encryptPage(data) {
if (!this.cryptoKey)
throw new Error('Crypto not initialized');
const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH$1));
// 传 TypedArray 视图而非裸 ArrayBufferSubtleCrypto 通过 ArrayBuffer.isView 检查,
// 对跨 realm / 跨 vm 环境的 ArrayBuffer 兼容(Node 18/20 的 webcrypto 对裸 ArrayBuffer 检查严格)
const plain = (data instanceof Uint8Array ? data : new Uint8Array(data));
const ciphertext = await crypto.subtle.encrypt({ name: ALGO, iv }, this.cryptoKey, plain);
return { iv: iv, data: ciphertext };
}
async decryptPage(iv, data) {
if (!this.cryptoKey)
throw new Error('Crypto not initialized');
const ciphertext = (data instanceof Uint8Array ? data : new Uint8Array(data));
return crypto.subtle.decrypt({ name: ALGO, iv }, this.cryptoKey, ciphertext);
}
close() {
this.cryptoKey = null;
this._enabled = false;
}
}
/**
* AriaEngine Encrypted Backend — 全库透明 AES-256-GCM 加密后端
* @module engine/aria/store/encrypted_backend
*
* 装饰器模式包装底层 IStorageBackend
* - 写入时加密(每个 value 独立随机 IV:格式 [12B IV][AES-GCM ciphertext]
* - 读取时解密(GCM 认证标签同时保证完整性)
* - WAL / SSTable / Schema / 元数据 全部密文存储(除密钥元数据本身)
*
* 密钥管理:
* - PBKDF2-SHA256100000 迭代)从密码派生 AES-256-GCM 密钥
* - `__aria_keymeta` 明文保存 { salt, verifier }
* - salt:PBKDF2 盐(重启后用同一密码重新派生密钥)
* - verifier:对固定明文加密的密文(打开时解密验证密码正确性)
* - 密码错误 → GCM 认证失败 → 抛 ARIA_DECRYPT_ERROR
*
* 依赖浏览器/Node 的 WebCryptojest.setup.js 已提供 polyfill)。
*/
const IV_LENGTH = 12;
const KEYMETA_KEY = '__aria_keymeta';
/** verifier 固定明文(无数据泄露风险) */
const VERIFIER_PLAIN = 'metona-sqlark-encryption-verifier';
// ---------------------------------------------------------------------------
// Base64 工具(浏览器 btoa/atob 与 Node Buffer 双环境)
// ---------------------------------------------------------------------------
function bytesToBase64(bytes) {
if (typeof Buffer !== 'undefined') {
return Buffer.from(bytes).toString('base64');
}
let bin = '';
for (let i = 0; i < bytes.byteLength; i++)
bin += String.fromCharCode(bytes[i]);
return btoa(bin);
}
function base64ToBytes(b64) {
if (typeof Buffer !== 'undefined') {
return new Uint8Array(Buffer.from(b64, 'base64'));
}
const bin = atob(b64);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++)
bytes[i] = bin.charCodeAt(i);
return bytes;
}
// ---------------------------------------------------------------------------
// EncryptedBackend
// ---------------------------------------------------------------------------
class EncryptedBackend {
constructor(inner, password) {
this.inner = inner;
this.password = password;
this.crypto = new CryptoManager();
this.opened = false;
if (!password || password.length === 0) {
throw new DatabaseError('Encryption password must not be empty', 'ARIA_ENCRYPT_CONFIG_ERROR');
}
}
/** 底层后端(测试/调试用) */
getInner() {
return this.inner;
}
/** 加密是否已初始化(打开并验证/创建密钥后为 true) */
isCryptoReady() {
return this.crypto.enabled;
}
async open(name) {
if (!this.inner.isOpen()) {
await this.inner.open(name);
}
const raw = await this.inner.read(KEYMETA_KEY);
if (raw) {
// 已有密钥元数据:用持久化 salt 重新派生并验证密码
let meta;
try {
meta = JSON.parse(new TextDecoder().decode(raw));
}
catch {
throw new DatabaseError('Corrupted encryption key metadata', 'ARIA_DECRYPT_ERROR');
}
if (!meta.salt || !meta.verifier) {
throw new DatabaseError('Corrupted encryption key metadata', 'ARIA_DECRYPT_ERROR');
}
const salt = base64ToBytes(meta.salt);
await this.crypto.init(this.password, salt);
const verifierBytes = base64ToBytes(meta.verifier);
if (verifierBytes.byteLength <= IV_LENGTH) {
throw new DatabaseError('Corrupted encryption key metadata', 'ARIA_DECRYPT_ERROR');
}
const iv = verifierBytes.subarray(0, IV_LENGTH);
const ciphertext = verifierBytes.subarray(IV_LENGTH);
try {
await this.crypto.decryptPage(iv, ciphertext);
}
catch {
// GCM 认证失败:密码错误(或密钥元数据被篡改)
throw new DatabaseError('Decryption failed: wrong password or corrupted key metadata', 'ARIA_DECRYPT_ERROR');
}
}
else {
// 无 keymeta:若库中已存在其他数据 → 明文旧库或 keymeta 丢失,拒绝以加密模式打开
const keys = await this.inner.listKeys();
if (keys.some((k) => k !== KEYMETA_KEY)) {
throw new DatabaseError('Cannot open with encryption: existing database has no key metadata ' +
'(database was created without encryption, or key metadata was lost)', 'ARIA_ENCRYPT_CONFIG_ERROR');
}
// 新库:生成随机 salt + 派生密钥 + 写入 verifier
const salt = await this.crypto.init(this.password);
const enc = await this.crypto.encryptPage(new TextEncoder().encode(VERIFIER_PLAIN).buffer);
const verifier = new Uint8Array(IV_LENGTH + enc.data.byteLength);
verifier.set(enc.iv, 0);
verifier.set(new Uint8Array(enc.data), IV_LENGTH);
const keymeta = JSON.stringify({
salt: bytesToBase64(salt),
verifier: bytesToBase64(verifier),
});
await this.inner.write(KEYMETA_KEY, new TextEncoder().encode(keymeta).buffer);
}
this.opened = true;
}
async close() {
await this.inner.close();
this.crypto.close();
this.opened = false;
}
isOpen() {
return this.opened;
}
async read(key) {
this.ensureReady();
const raw = await this.inner.read(key);
if (raw === null)
return null;
return this.decrypt(raw);
}
async write(key, data) {
this.ensureReady();
await this.inner.write(key, await this.encrypt(data));
}
async writeMany(entries) {
this.ensureReady();
const encrypted = {};
for (const [key, data] of Object.entries(entries)) {
encrypted[key] = await this.encrypt(data);
}
await this.inner.writeMany(encrypted);
}
async delete(key) {
this.ensureReady();
await this.inner.delete(key);
}
async deleteMany(keys) {
this.ensureReady();
await this.inner.deleteMany(keys);
}
async listKeys() {
this.ensureReady();
return this.inner.listKeys();
}
async exists(key) {
this.ensureReady();
return this.inner.exists(key);
}
async clear() {
// 清空全部数据但保留密钥元数据 —— clearAll 语义:库本身保留,密码不失效
const keys = await this.inner.listKeys();
const toDelete = keys.filter((k) => k !== KEYMETA_KEY);
if (toDelete.length > 0) {
await this.inner.deleteMany(toDelete);
}
}
// -----------------------------------------------------------------------
// 内部
// -----------------------------------------------------------------------
ensureReady() {
if (!this.opened)
throw new DatabaseError('EncryptedBackend not opened', 'ARIA_DB_NOT_OPEN');
if (!this.crypto.enabled)
throw new DatabaseError('EncryptedBackend key not initialized', 'ARIA_DECRYPT_ERROR');
}
/** 加密单块数据:[IV(12)][ciphertext] */
async encrypt(data) {
const enc = await this.crypto.encryptPage(data);
const out = new ArrayBuffer(IV_LENGTH + enc.data.byteLength);
const outBytes = new Uint8Array(out);
outBytes.set(enc.iv, 0);
outBytes.set(new Uint8Array(enc.data), IV_LENGTH);
return out;
}
/** 解密单块数据(GCM 认证失败抛错) */
async decrypt(data) {
const bytes = new Uint8Array(data);
if (bytes.byteLength <= IV_LENGTH) {
throw new DatabaseError('Corrupted encrypted data block (too short)', 'ARIA_DECRYPT_ERROR');
}
const iv = bytes.subarray(0, IV_LENGTH);
const ciphertext = bytes.subarray(IV_LENGTH);
try {
return await this.crypto.decryptPage(iv, ciphertext);
}
catch (error) {
if (error instanceof DatabaseError)
throw error;
throw new DatabaseError('Decryption failed (data corrupted or wrong key)', 'ARIA_DECRYPT_ERROR', error);
}
}
}
/**
* AriaEngine Page SSTable Store — SSTable 页面化物理存储
* @module engine/aria/store/page_sstable_store
*
* v0.4.5: 让 BufferPool/FileManager 真正接入 LSM 读写路径。
* SSTable 不再整体存为一个 backend value,而是切分为 4KB 页面:
* - 页面由 FileManager 分配 pageId,经 BufferPool 缓存(LRU 驱逐,脏页写回)
* - save 语义 = 数据已落盘:页面写入后逐个 flushPage(await 底层写)才返回,
* 保证 WAL checkpoint(截断)前 SSTable 数据真实持久化
* - 页面 ID 列表经 SSTableMeta.pageIds 持久化;旧 meta(无 pageIds)走整 value 读取
*
* 与 LSM 的 sstableCache(整文件 LRU)双层缓存并存:
* - BufferPool 缓存物理页面(跨 SSTable 共享、受 bufferPoolPages 上限约束)
* - LSM 缓存解析后的整文件字节(查询热点复用)
*/
class PageSSTableStore {
constructor(fileManager, bufferPool) {
this.fileManager = fileManager;
this.bufferPool = bufferPool;
/** SSTable id → 页面 ID 列表(save 时记录,saveMeta 时注入 meta */
this.pageIds = new Map();
}
/** 保存数据:切页 → 写入 BufferPool → 逐页落盘 → 记录 pageIds */
async save(id, data) {
const pageCount = Math.max(1, Math.ceil(data.byteLength / PAGE_SIZE));
const handles = await this.bufferPool.newPages(pageCount, PageType.DATA);
const ids = [];
for (let i = 0; i < pageCount; i++) {
const page = handles[i];
ids.push(page.pageId);
const dest = new Uint8Array(page.data);
dest.fill(0); // 清空(最后一页可能不满)
const slice = data.subarray(i * PAGE_SIZE, Math.min((i + 1) * PAGE_SIZE, data.byteLength));
dest.set(slice, 0);
page.dirty = true;
// save 语义 = 已持久化:立即落盘(WAL checkpoint 截断依赖此保证)
await this.bufferPool.flushPage(page.pageId);
this.bufferPool.unpin(page);
}
this.pageIds.set(id, ids);
}
/** 获取指定 SSTable 的页面 ID 列表(saveMeta 注入用) */
getPageIds(id) {
return this.pageIds.get(id);
}
/**
* 按页面 ID 列表读取并拼接为完整字节流。
* @param totalSize SSTable 真实大小(meta 持久化)——最后一页可能有 0 填充,按真实大小截断
* @returns 缺失页面/读取失败返回 null(调用方视为损坏并清理)
*/
async load(id, pageIds, totalSize) {
if (pageIds.length === 0)
return null;
const chunks = [];
for (const pageId of pageIds) {
const page = await this.bufferPool.getPage(pageId);
if (!page)
return null;
// 立即复制(后续驱逐安全)
chunks.push(new Uint8Array(page.data));
this.bufferPool.unpin(page);
}
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
const out = new Uint8Array(Math.min(total, totalSize));
let off = 0;
for (const c of chunks) {
const take = Math.min(c.byteLength, out.byteLength - off);
if (take <= 0)
break;
out.set(c.subarray(0, take), off);
off += take;
}
this.pageIds.delete(id);
return out;
}
/** 释放页面(删除物理页面文件 + 移出 BufferPool */
async delete(id, pageIds) {
for (const pageId of pageIds) {
this.bufferPool.removePage(pageId);
try {
await this.fileManager.freePageId(pageId);
}
catch { /* 清理失败不阻塞 */ }
}
this.pageIds.delete(id);
}
}
/**
* AriaEngine File Manager — 页面文件管理 + PageIO 实现
* @module engine/aria/store/file_manager
*
* 负责管理页面文件的生命周期:分配/释放页面 ID,读写页面。
*/
// ---------------------------------------------------------------------------
// FileManager (implements PageIO)
// ---------------------------------------------------------------------------
class FileManager {
constructor(backend) {
this.nextPageId = 0;
this.metaLoaded = false;
this.dbName = '';
this.backend = backend;
}
/** 初始化:从存储中读取元数据 */
async init(dbName) {
this.dbName = dbName;
const meta = await this.backend.read('__aria_meta');
let nextPageId = 1;
if (meta && meta instanceof ArrayBuffer && meta.byteLength >= 4) {
nextPageId = new DataView(meta).getUint32(0, false);
}
// v0.6.1-fix(P0): 崩溃一致性 —— allocatePageIds 的 saveMeta 可能在分配
// 页面后被崩溃中断(nextPageId 回退)。恢复时若按回退值继续分配,
// 页面 id 复用会覆盖旧页面;随后 compaction 按 meta.pageIds 删除"旧"SSTable
// 时误删被复用的新数据 → 崩溃恢复后静默丢数据(kv 后端 2 万行丢 75%)。
// 修复:以"现存最大页面 id + 1"为准(单调不回退,绝不复用已存在页面)。
const keys = await this.backend.listKeys();
for (const k of keys) {
if (k.startsWith('pg_')) {
const id = Number.parseInt(k.slice(3), 10);
if (!Number.isNaN(id) && id + 1 > nextPageId)
nextPageId = id + 1;
}
}
this.nextPageId = nextPageId;
if (!meta || !(meta instanceof ArrayBuffer) || meta.byteLength < 4 || nextPageId !== new DataView(meta).getUint32(0, false)) {
await this.saveMeta();
}
this.metaLoaded = true;
}
// ---- PageIO ----
async readPage(pageId) {
const key = `pg_${pageId}`;
const data = await this.backend.read(key);
if (!data) {
// v0.4.5: 页面总是先分配(allocatePageId 持久化)后写入 —— 读取缺失页面视为损坏
// (此前返回空页面会静默掩盖页面丢失,页面化 SSTable 依赖 null 触发自愈清理)
return null;
}
// 确保大小正确
if (data.byteLength < PAGE_SIZE) {
const padded = new ArrayBuffer(PAGE_SIZE);
new Uint8Array(padded).set(new Uint8Array(data));
return padded;
}
return data;
}
async writePage(pageId, data) {
const key = `pg_${pageId}`;
await this.backend.write(key, data);
}
async allocatePageId() {
const id = this.nextPageId++;
await this.saveMeta();
return id;
}
/** v0.4.5: 批量分配页面 ID(一次 meta 持久化,避免页面化 SSTable 保存时逐页写 meta */
async allocatePageIds(count) {
if (count <= 0)
return [];
const ids = [];
const start = this.nextPageId;
this.nextPageId += count;
for (let i = 0; i < count; i++)
ids.push(start + i);
await this.saveMeta();
return ids;
}
async freePageId(_pageId) {
// 简化实现:不回收 pageId
const key = `pg_${_pageId}`;
await this.backend.delete(key);
}
// ---- 辅助 ----
async saveMeta() {
const buf = new ArrayBuffer(8);
new DataView(buf).setUint32(0, this.nextPageId, false);
await this.backend.write('__aria_meta', buf);
}
/** 清空所有数据 */
async clearAll() {
await this.backend.clear();
this.nextPageId = 1;
await this.saveMeta();
}
}
/**
* AriaEngine MVCC — 多版本并发控制
* @module engine/aria/transaction/mvcc
*
* 实现快照隔离 (Snapshot Isolation)。
* 每个事务看到数据库在事务开始时的快照。
*/
// ---------------------------------------------------------------------------
// MVCCManager
// ---------------------------------------------------------------------------
class MVCCManager {
constructor() {
/** 所有行版本的存储:tableName.key → 版本链 */
this.versionStore = new Map();
/** 活跃事务表:txnId → TxnEntry */
this.activeTxns = new Map();
/** 事务 ID 计数器 */
this.nextTxnId = 1;
/** 全局提交序列号(用于可见性判断) */
this.globalCommitLsn = 0;
/**
* v0.4.2-fix: 每个事务写入的 tableKey 集合 —
* commit/rollback 只遍历本事务写过的 key,避免全库版本链扫描(大表事务 O(N) → O(写入数))
*/
this.txnWriteKeys = new Map();
}
// =======================================================================
// 事务管理
// =======================================================================
/** 开始一个事务,返回事务 ID */
beginTransaction() {
const txnId = this.nextTxnId++;
this.activeTxns.set(txnId, {
txnId,
state: TransactionState.ACTIVE,
snapshotLsn: this.globalCommitLsn,
startTime: Date.now(),
});
this.txnWriteKeys.set(txnId, new Set());
return txnId;
}
/** 提交事务 */
commitTransaction(txnId) {
const txn = this.activeTxns.get(txnId);
if (!txn)
throw new Error(`Transaction ${txnId} not found`);
txn.state = TransactionState.COMMITTED;
this.globalCommitLsn++;
// v0.6.3-fix: 已提交版本直接清理 —— 快照读取已移除(v0.5.1),版本链仅作
// 事务内 undo 记录(rollback/savepoint 用),提交后 LSM 持有权威数据。
// 此前 commit 仅标记 committed → versionStore 随写入量无限增长(行数据双份常驻)。
const writeKeys = this.txnWriteKeys.get(txnId);
if (writeKeys) {
for (const tableKey of writeKeys) {
const versions = this.versionStore.get(tableKey);
if (!versions)
continue;
const filtered = versions.filter((v) => v.txnId !== txnId);
if (filtered.length === 0) {
this.versionStore.delete(tableKey);
}
else {
this.versionStore.set(tableKey, filtered);
}
}
}
// 清理已提交事务的记录
this.activeTxns.delete(txnId);
this.txnWriteKeys.delete(txnId);
}
/** 回滚事务 */
rollbackTransaction(txnId) {
const txn = this.activeTxns.get(txnId);
if (!txn)
throw new Error(`Transaction ${txnId} not found`);
txn.state = TransactionState.ABORTED;
// v0.4.2-fix: 仅移除本事务写入的版本(此前遍历全库 versionStore
const writeKeys = this.txnWriteKeys.get(txnId);
if (writeKeys) {
for (const tableKey of writeKeys) {
const versions = this.versionStore.get(tableKey);
if (!versions)
continue;
const filtered = versions.filter((v) => v.txnId !== txnId);
if (filtered.length === 0) {
this.versionStore.delete(tableKey);
}
else {
this.versionStore.set(tableKey, filtered);
}
}
}
this.activeTxns.delete(txnId);
this.txnWriteKeys.delete(txnId);
}
// =======================================================================
// 版本读写
// =======================================================================
/**
* 写入一行(创建新版本)。
*/
writeVersion(tableName, key, data, txnId) {
const tableKey = `${tableName}.${key}`;
const versions = this.versionStore.get(tableKey) ?? [];
const newVersion = {
txnId,
data,
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
committed: false,
};
versions.push(newVersion);
this.versionStore.set(tableKey, versions);
// v0.4.2-fix: 记录本事务写过的 keycommit/rollback 精准清理)
this.txnWriteKeys.get(txnId)?.add(tableKey);
}
/**
* 删除一行(创建墓碑版本)。
*/
deleteVersion(tableName, key, txnId) {
this.writeVersion(tableName, key, { __mvcc_tombstone: true }, txnId);
}
/**
* v0.3.3: 丢弃指定事务的所有版本记录,但保留事务登记(Savepoint 回滚用)。
* 快照数据由调用方(引擎 txnSnapshot)负责恢复。
* v0.4.2-fix: 仅遍历本事务写过的 key(此前全库扫描)。
*/
discardVersions(txnId) {
const writeKeys = this.txnWriteKeys.get(txnId);
if (!writeKeys)
return;
for (const tableKey of writeKeys) {
const versions = this.versionStore.get(tableKey);
if (!versions)
continue;
const filtered = versions.filter((v) => v.txnId !== txnId);
if (filtered.length === 0) {
this.versionStore.delete(tableKey);
}
else {
this.versionStore.set(tableKey, filtered);
}
}
}
/**
* 清理过旧版本(GC)。
* 保留每个 key 的最新 N 个已提交版本。
*/
gc(maxVersionsPerKey = 100) {
for (const [tableKey, versions] of this.versionStore) {
if (versions.length <= maxVersionsPerKey)
continue;
// 保留最新的 maxVersionsPerKey 个版本
const pruned = versions.slice(versions.length - maxVersionsPerKey);
this.versionStore.set(tableKey, pruned);
}
}
/**
* 获取全局 LSN。
*/
getGlobalLSN() {
return this.globalCommitLsn;
}
}
/**
* AriaEngine Page Header — 页面头初始化
* @module engine/aria/page/header
*
* v0.4.5: 仅保留生产代码实际使用的 initPageHeader。
* 页面头部布局(大端序):
* [0-3] page_id u32
* [4] type u8
* [5-6] free_start u16
* [7-8] free_end u16
* [9-10] slot_count u16
* [11-14] checksum u32
* [15] reserved u8
*
* 注:free_start/free_end/slot_count/checksum 字段为历史行级页面格式遗留,
* 页面化 SSTable 使用页面原始字节区(跳过头部),字段保留以维持 16 字节头部对齐。
*/
/**
* 初始化新页面的 Header。
*/
function initPageHeader(buf, pageId, type) {
const view = new DataView(buf);
view.setUint32(0, pageId, false);
view.setUint8(4, type);
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
view.setUint16(9, 0, false); // slotCount = 0
view.setUint32(11, 0, false); // checksum = 0
view.setUint8(15, 0);
}
/**
* AriaEngine Page Format — 页面创建
* @module engine/aria/page/format
*
* v0.4.5: 仅保留生产代码实际使用的页面创建逻辑。
* Slot Directory / Tuple 编解码曾为行级页面存储设计,但从未接入 LSM 主路径,
* 属"宣称页面式但未实现"的半成品,已删除 —— SSTable 以 4KB 页面承载,
* 页面内容为原始字节切片,由 PageSSTableStore 管理)
*/
/** 创建一个新的空页面 */
function createPage(pageId, type) {
const data = new ArrayBuffer(PAGE_SIZE);
initPageHeader(data, pageId, type);
return {
pageId,
type,
data,
dirty: true,
pins: 0,
prev: null,
next: null,
lastAccess: Date.now(),
};
}
/**
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
* @module engine/aria/buffer/eviction
*/
// ---------------------------------------------------------------------------
// LRU 双向链表
// ---------------------------------------------------------------------------
/**
* LRU 链表管理器 — 双向链表,头部是 most recently used,尾部是 least recently used。
*/
class LRUList {
constructor() {
this.head = null;
this.tail = null;
this._size = 0;
}
get size() {
return this._size;
}
/** 将页面移到链表头部。如果是新页面则插入,已存在则移动。 */
moveToHead(page) {
// 如果已经在头部,无需操作
if (this.head === page)
return;
// 检测是否在链表中
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
if (inList) {
// 先从当前位置移除
this.detach(page);
}
else {
this._size++;
}
// 插入头部
page.prev = null;
page.next = this.head;
if (this.head) {
this.head.prev = page;
}
this.head = page;
if (!this.tail) {
this.tail = page;
}
}
/** 从链表中移除页面 */
remove(page) {
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
if (!inList)
return;
this.detach(page);
this._size = Math.max(0, this._size - 1);
}
/** 内部:只调整指针,不修改 _size */
detach(page) {
if (page.prev) {
page.prev.next = page.next;
}
else if (this.head === page) {
this.head = page.next;
}
if (page.next) {
page.next.prev = page.prev;
}
else if (this.tail === page) {
this.tail = page.prev;
}
page.prev = null;
page.next = null;
}
/** 清空链表 */
clear() {
this.head = null;
this.tail = null;
this._size = 0;
}
/** 获取 LRU 尾部(最久未使用的页面) */
getLRU() {
return this.tail;
}
}
/**
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
*/
class EvictionManager {
/**
* @param onEvict 驱逐脏页时的写回回调
* @param onRemove v0.6.3: 页面被驱逐时的回调(BufferPool 用于同步清理 pages Map
* 此前驱逐只清 LRU 链表,pages Map 无限增长 → 1MB 内存预算形同虚设)
*/
constructor(capacity, onEvict, onRemove) {
this.lru = new LRUList();
this.capacity = capacity;
this.onEvict = onEvict;
this.onRemove = onRemove;
}
/** 访问页面,更新 LRU */
access(page) {
page.lastAccess = Date.now();
this.lru.moveToHead(page);
}
/** 添加新页面到池中 */
add(page) {
this.access(page);
}
/** 移除指定页面 */
remove(page) {
this.lru.remove(page);
}
/**
* 驱逐页面直到池中有足够空间。
* 只驱逐未 pin 的干净页面(dirty=false)。
* 如果没有干净页面可驱逐,尝试刷脏页。
*/
async evictIfNeeded(count) {
let evicted = 0;
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
// 找到可驱逐的页面
const victim = this.findEvictionCandidate();
if (!victim)
break;
// 脏页先刷盘
if (victim.dirty) {
await this.onEvict(victim);
victim.dirty = false;
}
this.lru.remove(victim);
this.onRemove?.(victim);
evicted++;
}
return evicted;
}
/** 查找驱逐候选(优先干净页面,然后最久未用的脏页) */
findEvictionCandidate() {
// 先从尾部找未 pin 的干净页面
let current = this.lru.getLRU();
while (current) {
if (current.pins === 0 && !current.dirty)
return current;
current = current.prev;
}
// 没有干净页,找未 pin 的脏页
current = this.lru.getLRU();
while (current) {
if (current.pins === 0)
return current;
current = current.prev;
}
return null;
}
/** 清空 */
clear() {
this.lru.clear();
}
}
/**
* AriaEngine Buffer Pool — 页面缓存池
* @module engine/aria/buffer/pool
*
* 管理固定数量页面的 LRU 缓存,减少对底层储存的访问。
*/
// ---------------------------------------------------------------------------
// Buffer Pool
// ---------------------------------------------------------------------------
class BufferPool {
constructor(pageIO, capacity = DEFAULT_BUFFER_POOL_PAGES) {
this.pages = new Map();
this.nextPageId = 0;
this.pageIO = pageIO;
this.eviction = new EvictionManager(capacity, async (page) => {
if (page.dirty) {
await this.pageIO.writePage(page.pageId, page.data);
page.dirty = false;
}
}, (page) => {
// v0.6.3-fix: 驱逐时同步从 pages Map 移除 —— 此前仅清 LRU 链表,
// pages Map 保留全部历史页面 → 内存无限增长(1MB 预算失效)
this.pages.delete(page.pageId);
});
}
// -----------------------------------------------------------------------
// 页面获取
// -----------------------------------------------------------------------
/**
* 获取页面(必要时从磁盘读取)。
* 返回 pin 的页面,使用完成后必须调用 unpin()。
*/
async getPage(pageId) {
// 已在池中
let page = this.pages.get(pageId);
if (page) {
this.eviction.access(page);
page.pins++;
return page;
}
// 需要从磁盘加载
const buffer = await this.pageIO.readPage(pageId);
if (!buffer)
return null;
// 确保有空间
await this.eviction.evictIfNeeded(1);
const type = new DataView(buffer).getUint8(4);
page = {
pageId,
type,
data: buffer,
dirty: false,
pins: 1,
prev: null,
next: null,
lastAccess: Date.now(),
};
this.pages.set(pageId, page);
this.eviction.add(page);
return page;
}
/**
* v0.4.5: 批量创建新页面(一次页面 ID 分配,页面化 SSTable 保存用)。
* 返回的页面均 pin 且 dirty=false(调用方写入后需 markDirty + flushPage)。
*/
async newPages(count, type = PageType.DATA) {
if (count <= 0)
return [];
let pageIds;
if (typeof this.pageIO.allocatePageIds === 'function') {
pageIds = await this.pageIO.allocatePageIds(count);
}
else {
pageIds = [];
for (let i = 0; i < count; i++)
pageIds.push(await this.pageIO.allocatePageId());
}
await this.eviction.evictIfNeeded(count);
const handles = [];
for (const pageId of pageIds) {
const page = createPage(pageId, type);
page.pins = 1;
this.pages.set(pageId, page);
this.eviction.add(page);
handles.push(page);
}
return handles;
}
/**
* 释放页面的 pin。
*/
unpin(page) {
if (page.pins > 0) {
page.pins--;
}
}
/**
* 标记页面为脏(需要写回)。
*/
markDirty(page) {
page.dirty = true;
}
/**
* 将脏页面刷新到磁盘。
*/
async flushPage(pageId) {
const page = this.pages.get(pageId);
if (page && page.dirty) {
await this.pageIO.writePage(pageId, page.data);
page.dirty = false;
}
}
/**
* 刷新所有脏页面。
*/
async flushAll() {
for (const [, page] of this.pages) {
if (page.dirty) {
await this.pageIO.writePage(page.pageId, page.data);
page.dirty = false;
}
}
}
/**
* 从缓存中删除指定页面(不刷盘)。
*/
removePage(pageId) {
const page = this.pages.get(pageId);
if (page) {
this.eviction.remove(page);
this.pages.delete(pageId);
}
}
/**
* 清空缓存池(先刷脏页)。
*/
async clear() {
await this.flushAll();
this.pages.clear();
this.eviction.clear();
}
}
/**
* AriaEngine LZ4 Compression — 简化 LZ4 压缩/解压
* @module engine/aria/compression/lz4
*
* v0.4.5 格式 v2:压缩流前增加 4 字节原始大小头(LE u32),
* 解压不再依赖外部估算(高压缩率数据下 buf.length*2 估算不足会截断)。
* 旧版压缩数据(无头)视为损坏(compression 选项自 v0.2.6 起已声明不向后兼容)。
*
* Token 格式(1 字节):
* hi 4bit = litLen (0-15)
* lo 4bit = matchField (1-15, 实际匹配 = field+4)
*
* 字面量-匹配序列: [token] [litLen bytes] [2B LE offset]
* 末尾纯字面量: [token with lo=0] [litLen bytes] ← 仅在流末尾出现
*/
const MIN_MATCH = 4;
const MAX_MATCH = MIN_MATCH + 15; // 19,匹配长度上限
/** 原始大小头字节数 */
const HEADER_SIZE = 4;
function compressLZ4(input) {
// 空输入:仅头部(原始大小 0
if (input.byteLength === 0) {
const empty = new Uint8Array(HEADER_SIZE);
new DataView(empty.buffer).setUint32(0, 0, true);
return empty;
}
// 最坏情况:纯字面量分块输出 len/15 个 token + 末尾 token
// 上限:len + ceil(len/15) + 8(组合 token 的 offset 开销已包含在内)
const maxOut = input.byteLength + Math.ceil(input.byteLength / 15) + 8;
const out = new Uint8Array(maxOut);
let si = 0, di = 0;
let litStart = 0;
while (si < input.byteLength) {
// 搜索最长 backward match(截断到 MAX_MATCH,避免 token 字段溢出)
let bestLen = 0, bestOff = 0;
const searchStart = Math.max(0, si - 65535);
for (let p = searchStart; p < si; p++) {
let ml = 0;
while (si + ml < input.byteLength && p + ml < si &&
input[p + ml] === input[si + ml] && ml < MAX_MATCH)
ml++;
if (ml >= MIN_MATCH && ml > bestLen) {
bestLen = ml;
bestOff = si - p;
}
}
// 仅当匹配完整可编码(field 1-15)且字面量不超过 15 时才输出组合 token
if (bestLen > MIN_MATCH && (si - litStart) <= 15) {
const litLen = si - litStart;
const matchField = bestLen - MIN_MATCH; // 1..15
out[di++] = ((litLen & 0x0F) << 4) | (matchField & 0x0F);
for (let j = 0; j < litLen; j++)
out[di++] = input[litStart + j];
out[di++] = bestOff & 0xFF;
out[di++] = (bestOff >> 8) & 0xFF;
si += bestLen;
litStart = si;
}
else {
// 无匹配 / 匹配长度 4(field=0 有歧义)→ 继续累积字面量
si++;
// 字面量达到 15 字节上限:结清为纯字面量 token(lo=0),
// 否则后续组合 token 的字面量长度会超过 token 字段上限
if (si - litStart >= 15) {
out[di++] = (15 & 0x0F) << 4; // lo=0 无匹配
for (let j = 0; j < 15; j++)
out[di++] = input[litStart + j];
litStart = si;
}
}
}
// 输出末尾纯字面量(matchField=0,无 offset
let remaining = si - litStart;
while (remaining > 0) {
const chunk = Math.min(remaining, 15);
out[di++] = (chunk & 0x0F) << 4; // lo=0 表示无匹配/无 offset
for (let j = 0; j < chunk; j++)
out[di++] = input[litStart + j];
remaining -= chunk;
litStart += chunk;
}
// v0.4.5: 前置原始大小头,解压端自描述
const stream = out.slice(0, di);
const combined = new Uint8Array(HEADER_SIZE + stream.byteLength);
new DataView(combined.buffer).setUint32(0, input.byteLength, true);
combined.set(stream, HEADER_SIZE);
return combined;
}
function decompressLZ4(input, _originalSize) {
if (input.byteLength < HEADER_SIZE) {
throw new Error('LZ4 stream too short: missing header');
}
const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
const originalSize = view.getUint32(0, true);
if (originalSize === 0 && input.byteLength === HEADER_SIZE) {
return new Uint8Array(0); // 空输入
}
if (originalSize <= 0 || originalSize > 0x3fffffff) {
throw new Error('Invalid LZ4 header: bad original size');
}
const stream = input.subarray(HEADER_SIZE);
const out = new Uint8Array(originalSize);
let si = 0, di = 0;
while (si < stream.byteLength && di < originalSize) {
const token = stream[si++];
const litLen = (token >> 4) & 0x0F;
const matchField = token & 0x0F;
// 复制字面量
for (let i = 0; i < litLen && si < stream.byteLength && di < originalSize; i++) {
out[di++] = stream[si++];
}
// matchField=0:纯字面量 token(无 offset 无匹配)。
// 可能出现在流中任意位置(超长字面量分块输出),不能 break
if (matchField === 0)
continue;
// 组合 token:读取 offset + 复制匹配(可能自重叠)
if (si + 1 >= stream.byteLength)
break;
const offset = stream[si++] | (stream[si++] << 8);
const matchLen = matchField + MIN_MATCH;
for (let i = 0; i < matchLen && di < originalSize; i++) {
out[di] = out[di - offset];
di++;
}
}
return out;
}
/**
* AriaEngine — 自研页面式存储引擎主类
* @module engine/aria/index
*
* v0.4.1: 外键级联 + ALTER TABLE 重写 + clearAll 重置 + 崩溃恢复加固
*/
// ---------------------------------------------------------------------------
// AriaEngine
// ---------------------------------------------------------------------------
class AriaEngine {
constructor(config = {}) {
this.name = 'aria';
this.opened = false;
this.dbName = '';
// v0.4.5: 多标签页独占锁(Web Locks API,OPFS 等无事务后端防并发写)
this.dbLock = null;
// 表结构
this.schemas = new Map();
this.tablePKs = new Map();
this.opCounter = 0;
// 二级索引:table.colKey → LSM
this.secondaryIndexes = new Map();
// MVCC 事务
this.mvcc = new MVCCManager();
this.currentTxnId = null;
this.txnSnapshot = null;
this.gcCounter = 0;
// ---- Savepoint 嵌套事务 ----
this.savepoints = new Map();
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
}
// =======================================================================
// 生命周期
// =======================================================================
async open(dbName, _version) {
if (this.opened)
return;
// v0.4.2-fix: 引擎内部错误统一包装为 DatabaseErrorARIA_OPEN_ERROR),
// 应用层可拿到 code 分类处理,不再抛出原生 RangeError/TypeError
try {
await this.openInternal(dbName);
}
catch (error) {
// 打开失败:释放已获取的锁(避免锁泄漏阻塞其他标签页)
if (this.dbLock) {
try {
await this.dbLock.release();
}
catch { /* ignore */ }
this.dbLock = null;
}
if (error instanceof DatabaseError)
throw error;
throw new DatabaseError(`Failed to open AriaEngine database "${dbName}"`, 'ARIA_OPEN_ERROR', error);
}
}
/** open 内部实现(错误包装在 open 外层) */
async openInternal(dbName) {
this.dbName = dbName;
// v0.4.5: 多标签页独占锁(Web Locks)— 不支持的环境降级为无锁(文档注明)
const lock = new DatabaseLock();
this.dbLock = lock;
const lockAcquired = await lock.acquire(dbName);
if (!lockAcquired) {
// eslint-disable-next-line no-console
console.warn(`[AriaEngine] Web Locks API unavailable: no multi-tab protection for "${dbName}" ` +
'(open the same database in multiple tabs may corrupt data)');
}
// 1. 存储后端(可选全库加密包装)
let baseBackend;
if (this.config.storageBackend === 'opfs') {
baseBackend = new OPFSBackend();
}
else if (this.config.storageBackend === 'kv') {
// v0.6.1: 自研 KVStore 后端(aria 完全跑在自研存储栈上,不依赖浏览器 OPFS)
baseBackend = new KVStoreBackend();
}
else {
baseBackend = new MemoryBackend();
}
await baseBackend.open(dbName);
// v0.4.5: encryption 配置 → 透明加密封装(密码错误/数据损坏在 open 或首次读取时暴露)
if (this.config.encryption?.password) {
this.backend = new EncryptedBackend(baseBackend, this.config.encryption.password);
}
else {
this.backend = baseBackend;
// 反向检测:库中存在密钥元数据但未提供密码 → 拒绝打开(避免密文被当明文解析成空库)
if (await baseBackend.exists('__aria_keymeta')) {
await baseBackend.close();
throw new DatabaseError('Database is encrypted: provide encryption.password to open it', 'ARIA_ENCRYPT_REQUIRED');
}
}
await this.backend.open(dbName);
// 2a. FileManager (PageIO 实现) + Buffer Pool
this.fileManager = new FileManager(this.backend);
await this.fileManager.init(dbName);
this.bufferPool = new BufferPool(this.fileManager, this.config.bufferPoolPages);
// 2. 构建 SSTableStore
const sstableStore = this.createSSTableStore('main');
// 3. 初始化主 LSMPK 索引)
this.lsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
// SSTable 缓存上限 = BufferPool 页数 × 页面大小(默认 256 页 ≈ 1MB 可控内存)
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore,
});
// 4. 初始化 WAL
// v0.4.5: 分片式 WAL 存储(__wal_%06d.bin),序号内嵌记录字节流无需 count 键,
// append 单文件原子写;空洞检测截断;兼容旧格式 __wal_N + __wal_count
this.wal = new WAL(new SegmentedWALStore(this.backend), this.config.walEnabled, this.config.walSyncMode);
// 5. 恢复 Schema
await this.loadSchemas();
// v0.4.2-fix: 为 schema 中带 index/unique 标记的列重建二级索引 LSM。
// 此前重开只恢复 schema 不恢复索引 LSM → 索引查询静默回退全表、
// createIndex 因 colDef 已有标记直接 return → 索引永久缺失。
// 索引数据已持久化在独立命名空间(sst_idx_* / meta),init() 直接加载。
for (const [tableName, schema] of this.schemas) {
const pkCol = this.tablePKs.get(tableName);
for (const [colName, colDef] of Object.entries(schema.columns)) {
if ((colDef.index || colDef.unique) && colName !== pkCol) {
const idxKey = `${tableName}:idx:${colName}`;
if (!this.secondaryIndexes.has(idxKey)) {
const idxLsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore: this.createSSTableStore(`idx_${tableName}_${colName}`),
});
await idxLsm.init();
this.secondaryIndexes.set(idxKey, idxLsm);
}
}
}
}
// 6. 初始化 LSM(加载 SSTable 元数据)
await this.lsm.init();
// 7. WAL 恢复(两阶段:先扫描事务边界,仅回放已提交事务)
const committedTxns = new Set();
const allRecords = [];
await this.wal.recover((r) => allRecords.push(r));
// 第一遍:确定已提交事务
for (const r of allRecords) {
if (r.type === WALRecordType.COMMIT)
committedTxns.add(r.txnId);
if (r.type === WALRecordType.ROLLBACK)
committedTxns.delete(r.txnId);
}
// 第二遍:仅应用 txnId==0(非事务)或已提交事务的数据
for (const r of allRecords) {
if (r.txnId === 0 || committedTxns.has(r.txnId)) {
if (r.type === WALRecordType.DROP_TABLE) {
// v0.3.3: DROP_TABLE 回放(异步:需预加载 SSTable 后清除残留数据)
await this.applyDropTableRecovery(r.tableName);
}
else {
this.applyWALRecord(r);
}
}
}
// v0.3.3: 恢复完成后将回放数据落盘并截断 WAL,
// 避免每次重启重复回放 + WAL 无限膨胀
if (allRecords.length > 0) {
await this.lsm.flush();
await this.wal.checkpoint();
// v0.4.2-fix: WAL 回放只更新主 LSM,二级索引 LSM 未同步 →
// 崩溃前最后一批写入的索引缺失,重开时索引查询丢行。
// 恢复后全量重建所有表的二级索引(幂等)。
for (const tableName of this.schemas.keys()) {
await this.reindexTableInternal(tableName);
}
}
// 8. Checkpoint Manager(接入 WAL 大小阈值)
// v0.4.2-fix: 事务活跃时 checkpoint 不得截断 WAL —
// 否则 BEGIN/INSERT 记录被截断,COMMIT 后崩溃恢复丢失整个事务数据
this.checkpointManager = new CheckpointManager(this.lsm, {
checkpoint: async () => {
if (this.currentTxnId)
return;
await this.wal.checkpoint();
},
flush: async () => {
if (this.currentTxnId)
return;
await this.wal.flush();
},
getBufferedBytes: () => this.wal.getBufferedBytes(),
getBufferedCount: () => this.wal.getBufferedCount(),
}, {
flushAll: async () => {
// v0.6.1-fix: checkpoint 必须同时落盘二级索引 LSM —
// 此前只 flush 主 LSMcheckpoint 截断 WAL 后崩溃时索引 memtable 未落盘、
// WAL 为空跳过重建 → 二级索引静默丢失最后一批条目(生产数据一致性问题)
await this.lsm.flush();
for (const idxLsm of this.secondaryIndexes.values()) {
await idxLsm.flush();
}
},
}, this.config.checkpointInterval, this.config.walSizeThreshold);
this.opened = true;
}
async close() {
if (!this.opened)
return;
await this.persistSchemas();
await this.lsm.flush();
// v0.4.2-fix: 同步落盘全部二级索引 LSM — 此前只 flush 主 LSM
// 优雅关闭后索引 memtable 未落盘 → 重开索引为空 → 索引查询返回空结果
for (const idxLsm of this.secondaryIndexes.values()) {
await idxLsm.flush();
}
// v0.4.5: 页面化存储 — 落盘全部脏页(save 已逐页落盘,此处兜底)
await this.bufferPool.flushAll();
await this.wal.flush();
// v0.4.2-fix: close 前 checkpoint(截断 WAL)—
// 此前只 flush 不截断,下次打开会重放全部历史 WAL 记录(含已落盘 SSTable 的数据),
// 重复解析/重复 put 拖慢启动,并与恢复后 flush+checkpoint 竞争放大丢数据
await this.wal.checkpoint();
await this.backend.close();
// v0.4.5: 释放多标签页独占锁(等待锁真正归还)
if (this.dbLock) {
await this.dbLock.release();
this.dbLock = null;
}
// v0.4.2-fix: 清空运行期状态(此前 close 后 mvcc/txn 残留,
// 重开时 beginTransaction 报 TX_ACTIVE 或读到陈旧快照)
this.schemas.clear();
this.tablePKs.clear();
this.secondaryIndexes.clear();
this.mvcc = new MVCCManager();
this.currentTxnId = null;
this.txnSnapshot = null;
this.savepoints.clear();
this.opCounter = 0;
this.opened = false;
}
/**
* v0.4.2-fix: 崩溃恢复/自愈 — 校验并移除损坏 SSTable、截断 WAL、重建二级索引。
* v0.4.5 增强:清理 OPFS 残留临时文件、清理孤儿页面(meta 未引用的 pg_ 文件)。
* 应用层检测到异常后调用,无需删库重建。
*/
async repair() {
this.ensureOpen();
// v0.6.0-fix: 先清页面缓存再校验 — 缓存中的"完好页面"会掩盖磁盘损坏
await this.bufferPool.clear();
// 1. 校验全部 SSTable,移除残缺项(打开时已做一次,此处兜底运行期损坏)
const removed = await this.lsm.validateAll();
// 2. 将 WAL 残留数据落盘并截断,避免无限重放(含空洞截断落地)
await this.lsm.flush();
await this.wal.checkpoint();
// 3. 重建所有表的二级索引(修复索引与主数据不一致)
for (const tableName of this.schemas.keys()) {
await this.reindexTable(tableName);
}
// v0.4.5: 4. 清理 OPFS 残留临时文件(.crswap/.tmp
const backendAny = this.backend;
if (typeof backendAny.cleanupStaleFiles === 'function') {
try {
await backendAny.cleanupStaleFiles();
}
catch { /* 清理失败不阻塞 */ }
}
// v0.4.5: 5. 清理孤儿页面(所有 LSM 命名空间 meta 均未引用的 pg_ 文件)
await this.cleanupOrphanPages();
if (removed > 0) {
// eslint-disable-next-line no-console
console.warn(`[AriaEngine] repair: removed ${removed} corrupted SSTable(s)`);
}
}
/**
* v0.4.5: 清理孤儿页面 — 扫描全部 pg_* 文件,未被任何 LSM 命名空间 meta 引用的删除。
* 孤儿页面来自:崩溃中断的 compaction/删除流程(旧 SSTable 页面残留)。
*/
async cleanupOrphanPages() {
const keys = await this.backend.listKeys();
const pgKeys = keys.filter((k) => /^pg_\d+$/.test(k));
if (pgKeys.length === 0)
return;
const used = new Set();
const collectMeta = async (ns) => {
const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`;
const raw = await this.backend.read(META_KEY);
if (!raw)
return;
try {
const metas = JSON.parse(new TextDecoder().decode(raw));
for (const m of metas) {
if (m.pageIds) {
for (const pid of m.pageIds)
used.add(pid);
}
}
}
catch { /* 损坏的 meta 忽略(validateAll 已处理) */ }
};
await collectMeta('main');
// 收集全部二级索引命名空间
for (const [tableName, schema] of this.schemas) {
for (const [colName, colDef] of Object.entries(schema.columns)) {
if (colDef.index || colDef.unique) {
await collectMeta(`idx_${tableName}_${colName}`);
}
}
}
const orphanIds = pgKeys
.map((k) => Number(k.slice('pg_'.length)))
.filter((pid) => !used.has(pid));
if (orphanIds.length > 0) {
await this.backend.deleteMany(orphanIds.map((pid) => `pg_${pid}`));
}
}
/**
* v0.4.1: 重置数据库 — 清空全部数据与表结构(演示页刷新/重新初始化用)。
* 清空存储后端、LSM、WAL、MVCC 与二级索引,后续可继续使用本实例。
*/
async clearAll() {
this.ensureOpen();
// 清空存储后端(页面文件 / WAL 记录 / schema 记录 / 元数据)
await this.backend.clear();
// v0.4.5: 清空页面缓存与页面 ID 分配状态
await this.bufferPool.clear();
await this.fileManager.clearAll();
this.schemas.clear();
this.tablePKs.clear();
this.secondaryIndexes.clear();
this.lsm.clear();
this.mvcc = new MVCCManager();
this.currentTxnId = null;
this.txnSnapshot = null;
this.savepoints.clear();
this.opCounter = 0;
// 持久化空 schema(防止旧 schema 记录残留)
await this.persistSchemas();
// 重置 WAL 状态(backend.clear 已清记录,同步内存计数)
await this.wal.checkpoint();
}
isOpen() { return this.opened; }
// ---- v0.4.2-fix: 库内元数据(迁移版本持久化用) ----
async getMeta(key) {
const raw = await this.backend.read(`__meta_${key}`);
return raw ? new TextDecoder().decode(raw) : null;
}
async setMeta(key, value) {
await this.backend.write(`__meta_${key}`, new TextEncoder().encode(value).buffer);
}
// =======================================================================
// 表管理
// =======================================================================
async createTable(schema) {
this.ensureOpen();
// v0.4.2-fix: Aria 事务中 DDL 显式拒绝(事务快照只覆盖行数据,
// 结构变更无法回滚;Memory/IndexedDB 引擎快照可回滚,行为不一致 → 明确报错而非静默)
this.ensureNoDDLInTransaction('CREATE TABLE');
if (this.schemas.has(schema.name)) {
throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS');
}
this.schemas.set(schema.name, schema);
this.tablePKs.set(schema.name, this.getPK(schema));
// 为索引列创建二级索引 LSM(每个索引使用独立命名空间的 SSTableStore,避免 id/meta 冲突)
// v0.3.3: 主键列不建冗余二级索引(主 LSM 本身就是 PK 索引,范围查询走前缀扫描)
for (const [colName, colDef] of Object.entries(schema.columns)) {
if (colDef.index || colDef.unique) {
const idxKey = `${schema.name}:idx:${colName}`;
if (!this.secondaryIndexes.has(idxKey)) {
const idxLsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore: this.createSSTableStore(`idx_${schema.name}_${colName}`),
});
await idxLsm.init();
this.secondaryIndexes.set(idxKey, idxLsm);
}
}
}
await this.persistSchemas();
await this.wal.append({
type: WALRecordType.CREATE_TABLE,
txnId: 0,
tableName: schema.name,
key: '',
data: { schema: JSON.stringify(schema) },
});
}
async dropTable(tableName) {
this.ensureOpen();
this.ensureNoDDLInTransaction('DROP TABLE');
this.ensureTable(tableName);
// 删除表中所有行
const rows = await this.getAllRows(tableName);
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName);
this.lsm.delete(`${tableName}:${row[pkCol]}`);
}
// v0.4.2-fix: 清理该表的全部二级索引 LSM 与持久化文件 —
// 此前残留孤儿索引,重建同名表后旧索引数据污染新表(索引查询返回错误行)
await this.cleanupTableIndexes(tableName);
this.schemas.delete(tableName);
this.tablePKs.delete(tableName);
await this.persistSchemas();
await this.wal.append({
type: WALRecordType.DROP_TABLE,
txnId: 0,
tableName,
key: '',
});
}
/**
* v0.4.2-fix: 清理指定表的全部二级索引 LSM(内存 + 存储文件 + meta)。
* dropTable / DROP_TABLE 恢复 / alterTable DROP 索引列 共用。
*/
async cleanupTableIndexes(tableName) {
const prefix = `${tableName}:idx:`;
const toDelete = [];
for (const [idxKey, idxLsm] of this.secondaryIndexes) {
if (!idxKey.startsWith(prefix))
continue;
toDelete.push(idxKey);
try {
await idxLsm.clear();
}
catch { /* 清理失败不阻塞 */ }
}
for (const idxKey of toDelete) {
this.secondaryIndexes.delete(idxKey);
}
}
async hasTable(tableName) {
// v0.7.1: 未 open 防护统一(此前与 KVStoreEngine 不一致:返回空而非报错)
this.ensureOpen();
return this.schemas.has(tableName);
}
async getTableNames() {
this.ensureOpen();
return Array.from(this.schemas.keys());
}
async getTableSchema(tableName) {
this.ensureOpen();
return this.schemas.get(tableName) ?? null;
}
// =======================================================================
// CRUD
// =======================================================================
async insert(tableName, rows) {
this.ensureOpen();
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const pkCol = this.tablePKs.get(tableName);
const pks = [];
// v0.3.1: 批量 WAL 写入(组提交),一次 insert 合并为一次落盘
const walRecords = [];
// v0.6.1-perf: 批量预加载本批 PK 涉及的 SSTable(一次 drainChain)。
// 此前循环内逐行 prefetchKeys —— 每行 await drainChain 排空后台链,
// 后台 compaction 在链上数秒时每行阻塞数秒 → 大数据量插入性能悬崖
// 10 万行 kv 后端从 5ms/批暴跌到 8~11s/批)。批内新数据在 memtable
// 或 flush 产物(自动入缓存),循环内 lsm.get 始终完整。
//
// v0.6.2: 整批预校验(验证失败整批不落库,语义更原子)+ 唯一约束检查。
const uniqueCols = this.uniqueColumns(tableName, schema);
const validatedRows = [];
for (const row of rows) {
const validated = this.validateRow(schema, row);
const pkValue = String(validated[pkCol]);
validatedRows.push({ row: validated, pkValue, key: `${tableName}:${pkValue}` });
}
await this.lsm.prefetchKeys(validatedRows.map((v) => v.key));
// v0.6.2: 唯一约束 — 批量预加载本批唯一列涉及的索引范围(一次 drainChain
for (const colName of uniqueCols) {
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`);
await idxLsm.prefetchPrefixRanges(validatedRows
.map((v) => {
const val = v.row[colName];
if (val === undefined || val === null)
return null;
const p = `${String(val)}:`;
return [p, `${p}\uffff`];
})
.filter((r) => r !== null));
}
// v0.6.2: 唯一性整批预检(批内互查 + 索引查)—— 失败整批不落库(原子语义)
const batchUnique = new Map();
for (const { row: validated, pkValue } of validatedRows) {
for (const colName of uniqueCols) {
const val = validated[colName];
if (val === undefined || val === null)
continue;
const v = String(val);
let seen = batchUnique.get(colName);
if (!seen) {
seen = new Set();
batchUnique.set(colName, seen);
}
if (seen.has(v)) {
throw new DatabaseError(`Unique constraint violation on column "${colName}" in table "${tableName}"`, 'UNIQUE_VIOLATION');
}
seen.add(v);
this.checkUniqueSync(tableName, [colName], validated, pkValue);
}
}
for (const { row: validated, pkValue, key } of validatedRows) {
// Check duplicate in LSM + transaction snapshot
const existing = this.currentTxnId
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
: this.lsm.get(key);
if (existing && !existing.__txn_deleted) {
throw new DatabaseError(`Duplicate primary key "${pkValue}" in table "${tableName}"`, 'DUPLICATE_KEY');
}
if (this.currentTxnId && this.txnSnapshot) {
// Within transaction: buffer to snapshot + MVCC version chain
this.txnSnapshot.set(key, validated);
this.mvcc.writeVersion(tableName, pkValue, validated, this.currentTxnId);
}
else {
// Direct write to LSM (PK index)
this.lsm.put(key, validated);
}
// 更新二级索引
this.updateSecondaryIndexes(tableName, pkValue, validated, null);
pks.push(pkValue);
walRecords.push({
type: WALRecordType.INSERT,
txnId: this.currentTxnId ?? 0,
tableName,
key: pkValue,
data: validated,
});
}
await this.wal.appendBatch(walRecords);
this.opCounter += rows.length;
this.checkMemoryBudget();
await this.checkpointManager.tick();
this.tryGC();
return pks;
}
async find(tableName, query) {
this.ensureOpen();
this.ensureTable(tableName);
let rows;
// Try index lookup
const fastPath = await this.tryIndexLookup(tableName, query);
if (fastPath !== null) {
rows = fastPath;
}
else {
rows = await this.getAllRows(tableName);
}
// v0.3.3: 事务内合并未提交快照(统一在 mergeTxnSnapshot 处理)
rows = this.mergeTxnSnapshot(tableName, rows);
// WHERE filter
if (query.where && Object.keys(query.where).length > 0) {
rows = rows.filter((row) => matchWhere(row, query.where));
}
// ORDER
if (query.orderBy && query.orderBy.length > 0) {
rows = applyOrderBy(rows, query.orderBy);
}
// LIMIT/OFFSET
const offset = query.offset ?? 0;
const limit = query.limit ?? rows.length;
rows = rows.slice(offset, offset + limit);
// Column projection
if (query.columns && query.columns.length > 0 && query.columns[0] !== '*') {
rows = rows.map((row) => projectColumns(row, query.columns));
}
// 查询完成,回收查询期间的临时缓存超限
this.trimAllCaches();
return rows;
}
async update(tableName, query, updates) {
this.ensureOpen();
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const rows = await this.getAllRows(tableName);
let count = 0;
// v0.3.1: 批量 WAL 写入(组提交)
const walRecords = [];
// v0.4.2-fix: ON UPDATE 级联环路保护
const visited = new Set();
// v0.7.2: undefined 值视为"不更新该列"(保留旧值),null 显式置空
const cleanUpdates = stripUndefinedUpdates(updates);
// v0.6.2: 唯一约束 — 批量预加载本批更新涉及的唯一列索引范围(一次 drainChain
const uniqueCols = this.uniqueColumns(tableName, schema);
for (const colName of uniqueCols) {
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`);
const ranges = [];
if (cleanUpdates[colName] !== undefined && cleanUpdates[colName] !== null) {
const p = `${String(cleanUpdates[colName])}:`;
ranges.push([p, `${p}\uffff`]);
}
else if (!(colName in cleanUpdates)) {
for (const row of rows) {
const val = row[colName];
if (val === undefined || val === null)
continue;
const p = `${String(val)}:`;
ranges.push([p, `${p}\uffff`]);
}
}
await idxLsm.prefetchPrefixRanges(ranges);
}
// v0.7.2: 语句级原子性 — 两阶段(先全量预检,后执行)。
// 此前逐行"校验+写入":第 N 行唯一冲突/校验失败抛错时,前 N-1 行已写入
// 且其 WAL 记录随 appendBatch 一起丢失 → 内存已改、WAL 无记录、调用方已收到错误
// (无事务下语句级部分提交 + 崩溃后进一步不一致)。
const planned = [];
const batchUnique = new Map();
// 阶段 1:全量预检(任何一行失败 → 整条语句不执行)
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName);
const key = `${tableName}:${row[pkCol]}`;
if (query.where && Object.keys(query.where).length > 0 && !matchWhere(row, query.where))
continue;
const updated = { ...row, ...cleanUpdates };
this.validateRow(schema, updated);
// 批内唯一互查(索引尚未更新,两行同时改到同一新值需要互查兜底)
this.checkBatchUnique(tableName, uniqueCols, updated, batchUnique);
// v0.6.2: 唯一约束检查(排除自身旧索引条目:主键变更时旧条目仍以旧键存在)
this.checkUniqueSync(tableName, uniqueCols, updated, String(row[pkCol]));
// v0.4.2-fix: 支持更新主键 — 删除旧键 + 落新键 + WAL 两条记录
const newPk = String(updated[pkCol]);
const pkChanged = newPk !== String(row[pkCol]);
// v0.6.2-fix(P0): 主键变更撞已有主键 → 抛 DUPLICATE_KEY
// (此前静默覆盖另一行丢数据;与 MemoryEngine 对齐)
if (pkChanged) {
const newKey = `${tableName}:${newPk}`;
const existing = this.currentTxnId
? (this.txnSnapshot?.get(newKey) ?? this.lsm.get(newKey))
: this.lsm.get(newKey);
if (existing && !existing.__txn_deleted) {
throw new DatabaseError(`Duplicate primary key "${newPk}" in table "${tableName}" (cannot update key to existing value)`, 'DUPLICATE_KEY');
}
}
planned.push({ row, pk: String(row[pkCol]), key, updated, newPk, pkChanged });
}
// 阶段 1b:主键变更 RESTRICT / SET NULL+required 预检(任何修改前)
for (const p of planned) {
if (p.pkChanged) {
await this.checkForeignKeyUpdateRestrict(tableName, p.pk, p.newPk);
}
}
// 阶段 2:执行(预检已通过,此阶段不再抛校验类错误)
for (const { row, pk, key, updated, newPk, pkChanged } of planned) {
if (pkChanged) {
// ON UPDATE 外键级联(RESTRICT 抛错 / CASCADE / SET NULL
await this.applyForeignKeyUpdateRules(tableName, pk, newPk, walRecords, visited);
}
if (this.currentTxnId && this.txnSnapshot) {
if (pkChanged) {
this.txnSnapshot.set(key, { __txn_deleted: true });
this.mvcc.deleteVersion(tableName, pk, this.currentTxnId);
}
this.txnSnapshot.set(`${tableName}:${newPk}`, updated);
this.mvcc.writeVersion(tableName, newPk, updated, this.currentTxnId);
}
else {
if (pkChanged)
this.lsm.delete(key);
this.lsm.put(`${tableName}:${newPk}`, updated);
}
count++;
if (pkChanged) {
walRecords.push({
type: WALRecordType.DELETE,
txnId: this.currentTxnId ?? 0,
tableName,
key: pk,
});
}
walRecords.push({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName,
key: newPk,
data: updated,
});
// 更新二级索引(主键变更时旧索引条目一并清理)
// v0.6.2-fix: 此前非主键更新不传旧行 → 旧索引条目残留
// (唯一性检查误报 / 索引存储膨胀);现在统一传旧行清理旧值
this.updateSecondaryIndexes(tableName, newPk, updated, row);
}
await this.wal.appendBatch(walRecords);
this.opCounter += count;
await this.checkpointManager.tick();
this.trimAllCaches();
return count;
}
/**
* v0.7.2: 批内唯一互查 — 两条行在同一语句中更新到同一唯一值时的兜底检查
* (阶段 1 中索引尚未反映本语句的变更)。
*/
checkBatchUnique(tableName, uniqueCols, updated, batchUnique) {
for (const colName of uniqueCols) {
const value = updated[colName];
if (value === undefined || value === null)
continue;
let seen = batchUnique.get(colName);
if (!seen) {
seen = new Set();
batchUnique.set(colName, seen);
}
if (seen.has(value)) {
throw new DatabaseError(`Unique constraint violation on column "${colName}" in table "${tableName}"`, 'UNIQUE_VIOLATION');
}
seen.add(value);
}
}
/**
* v0.7.2: ON UPDATE 外键预检 — 从 applyForeignKeyUpdateRules 提取(两阶段 update 用):
* RESTRICT 存在依赖行抛错;SET NULL 撞 required 列同样整体拒绝。
*/
async checkForeignKeyUpdateRestrict(tableName, oldPk, _newPk) {
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
if (colDef.onUpdate === 'RESTRICT' || (colDef.onUpdate === 'SET NULL' && colDef.required)) {
const refRows = await this.getAllRows(refTableName);
for (const refRow of refRows) {
if (String(refRow[colName]) === oldPk) {
const reason = colDef.onUpdate === 'RESTRICT'
? `foreign key "${colName}" in "${refTableName}" has dependent rows`
: `foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`;
throw new DatabaseError(`Cannot update "${tableName}" key "${oldPk}": ${reason}`, 'FOREIGN_KEY_VIOLATION');
}
}
}
}
}
}
/**
* v0.4.2-fix: ON UPDATE 外键级联 — 主键 oldPk → newPk 时处理引用表。
* RESTRICT 抛错 / CASCADE 更新 FK / SET NULL 置空(含索引与 WAL 记录)。
* 两阶段:先全量 RESTRICT 检查,再执行级联。
*/
async applyForeignKeyUpdateRules(tableName, oldPk, newPk, walRecords, visited) {
const visitKey = `${tableName}:${oldPk}`;
if (visited.has(visitKey))
return;
visited.add(visitKey);
// 阶段 1: RESTRICT 检查
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
if (colDef.onUpdate !== 'RESTRICT')
continue;
const refRows = await this.getAllRows(refTableName);
if (refRows.some((r) => String(r[colName]) === oldPk)) {
throw new DatabaseError(`Cannot update "${tableName}" key "${oldPk}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
}
}
// 阶段 2: CASCADE / SET NULL
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onUpdate)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
if (colDef.onUpdate !== 'CASCADE' && colDef.onUpdate !== 'SET NULL')
continue;
const refRows = await this.getAllRows(refTableName);
for (const refRow of refRows) {
if (String(refRow[colName]) !== oldPk)
continue;
const refPkCol = this.tablePKs.get(refTableName);
const refPk = String(refRow[refPkCol]);
const updatedRef = { ...refRow, [colName]: colDef.onUpdate === 'CASCADE' ? newPk : null };
const refKey = `${refTableName}:${refPk}`;
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(refKey, updatedRef);
this.mvcc.writeVersion(refTableName, refPk, updatedRef, this.currentTxnId);
}
else {
this.lsm.put(refKey, updatedRef);
}
this.updateSecondaryIndexes(refTableName, refPk, updatedRef, refRow);
walRecords.push({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName: refTableName,
key: refPk,
data: updatedRef,
});
}
}
}
}
async delete(tableName, query) {
this.ensureOpen();
this.ensureTable(tableName);
const rows = await this.getAllRows(tableName);
let count = 0;
// v0.3.1: 批量 WAL 写入(组提交)
const walRecords = [];
// v0.4.1: 外键级联(环路保护)
const visited = new Set();
// v0.6.3-fix: 级联两阶段 —— 先对全部匹配行做 RESTRICT 预检(沿 CASCADE 链递归),
// 否则第 N 行 RESTRICT 抛错时前 N-1 行的级联已执行 → 无事务部分级联(数据不一致)
const matchedPks = [];
for (const row of rows) {
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
matchedPks.push(String(row[this.tablePKs.get(tableName)]));
}
}
const restrictVisited = new Set();
for (const pkValue of matchedPks) {
await this.checkCascadeRestrict(tableName, pkValue, restrictVisited);
}
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName);
const key = `${tableName}:${row[pkCol]}`;
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
// v0.4.1: 外键规则(RESTRICT 抛错 / CASCADE 递归删 / SET NULL 置空)
count += await this.applyForeignKeyRules(tableName, String(row[pkCol]), walRecords, visited);
if (this.currentTxnId && this.txnSnapshot) {
// Buffer delete in snapshot + MVCC tombstone
this.txnSnapshot.set(key, { __txn_deleted: true });
this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId);
}
else {
this.lsm.delete(key);
}
count++;
walRecords.push({
type: WALRecordType.DELETE,
txnId: this.currentTxnId ?? 0,
tableName,
key: String(row[pkCol]),
});
// 移除二级索引
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
}
}
await this.wal.appendBatch(walRecords);
this.opCounter += count;
await this.checkpointManager.tick();
this.trimAllCaches();
return count;
}
/**
* v0.6.3: RESTRICT 预检(delete 级联两阶段之一,与 MemoryEngine 对齐)。
* 递归沿 CASCADE 链检查引用表:RESTRICT 引用存在依赖行则抛 FOREIGN_KEY_VIOLATION。
*/
async checkCascadeRestrict(tableName, pkValue, visited) {
const visitKey = `${tableName}:${pkValue}`;
if (visited.has(visitKey))
return;
visited.add(visitKey);
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onDelete)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refRows = await this.getAllRows(refTableName);
const matched = refRows.filter((r) => String(r[colName]) === pkValue);
if (colDef.onDelete === 'RESTRICT' && matched.length > 0) {
throw new DatabaseError(`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
// v0.7.2: SET NULL 到 required 列违反约束 —— 预检阶段整体拒绝
if (colDef.onDelete === 'SET NULL' && colDef.required && matched.length > 0) {
throw new DatabaseError(`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" is required (SET NULL violates constraint)`, 'FOREIGN_KEY_VIOLATION');
}
if (colDef.onDelete === 'CASCADE') {
const refPkCol = this.tablePKs.get(refTableName);
for (const refRow of matched) {
await this.checkCascadeRestrict(refTableName, String(refRow[refPkCol]), visited);
}
}
}
}
}
/**
* v0.4.1: 外键级联规则 — 对齐 MemoryEngine.cascadeDelete 行为。
* 删除 tableName 主键为 pkValue 的行前,检查引用它的所有表:
* - RESTRICT: 存在引用行 → 抛 FOREIGN_KEY_VIOLATION
* - CASCADE: 递归删除引用行(含索引/WAL)
* - SET NULL: 引用行外键列置 null(含索引/WAL)
* @returns 级联影响的行数(CASCADE 删除行数 + SET NULL 更新行数)
*/
async applyForeignKeyRules(tableName, pkValue, walRecords, visited) {
let total = 0;
const visitKey = `${tableName}:${pkValue}`;
if (visited.has(visitKey))
return 0;
visited.add(visitKey);
for (const [refTableName, refSchema] of this.schemas) {
if (refTableName === tableName)
continue;
for (const [colName, colDef] of Object.entries(refSchema.columns)) {
if (!colDef.references || !colDef.onDelete)
continue;
const [refTable] = colDef.references.split('.');
if (refTable !== tableName)
continue;
const refRows = await this.getAllRows(refTableName);
const matched = refRows.filter((r) => String(r[colName]) === pkValue);
if (colDef.onDelete === 'RESTRICT' && matched.length > 0) {
throw new DatabaseError(`Cannot delete from "${tableName}": foreign key "${colName}" in "${refTableName}" has dependent rows`, 'FOREIGN_KEY_VIOLATION');
}
if (colDef.onDelete === 'CASCADE') {
const refPkCol = this.tablePKs.get(refTableName);
for (const refRow of matched) {
const refPk = String(refRow[refPkCol]);
// 递归级联(先处理更深层引用)
total += await this.applyForeignKeyRules(refTableName, refPk, walRecords, visited);
// 删除引用行
const refKey = `${refTableName}:${refPk}`;
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(refKey, { __txn_deleted: true });
this.mvcc.deleteVersion(refTableName, refPk, this.currentTxnId);
}
else {
this.lsm.delete(refKey);
}
this.updateSecondaryIndexes(refTableName, refPk, null, refRow);
walRecords.push({
type: WALRecordType.DELETE,
txnId: this.currentTxnId ?? 0,
tableName: refTableName,
key: refPk,
});
total++;
}
}
else if (colDef.onDelete === 'SET NULL') {
const refPkCol = this.tablePKs.get(refTableName);
for (const refRow of matched) {
const refPk = String(refRow[refPkCol]);
const updated = { ...refRow, [colName]: null };
const refKey = `${refTableName}:${refPk}`;
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(refKey, updated);
this.mvcc.writeVersion(refTableName, refPk, updated, this.currentTxnId);
}
else {
this.lsm.put(refKey, updated);
}
this.updateSecondaryIndexes(refTableName, refPk, updated, refRow);
walRecords.push({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName: refTableName,
key: refPk,
data: updated,
});
// 对齐 Memory 语义:SET NULL 不影响返回的删除行数
}
}
}
}
return total;
}
/**
* v0.4.0: 流式查询 — 逐行回调,不物化结果数组。
* 全表路径走 LSM rangeScanLazy 惰性扫描;索引等值/范围路径复用 tryIndexLookup。
* 事务中回退物化(快照合并需要全量行集)。
*/
async findStream(tableName, query, onRow) {
this.ensureOpen();
this.ensureTable(tableName);
const hasWhere = !!(query.where && Object.keys(query.where).length > 0);
const project = query.columns && query.columns.length > 0 && query.columns[0] !== '*'
? (row) => projectColumns(row, query.columns)
: null;
const limit = query.limit ?? Infinity;
const offset = query.offset ?? 0;
const pkCol = this.tablePKs.get(tableName);
const prefix = `${tableName}:`;
let count = 0;
let skipped = 0;
const emit = (row) => {
if (hasWhere && !matchWhere(row, query.where))
return true;
if (skipped < offset) {
skipped++;
return true;
}
onRow(project ? project(row) : row);
count++;
return count < limit;
};
if (this.currentTxnId && this.txnSnapshot) {
// 事务中:物化后逐行回调(快照合并需要全量行集)
const rows = await this.find(tableName, { ...query, orderBy: undefined, limit: undefined, offset: undefined });
for (const row of rows) {
onRow(project ? project(row) : row);
}
return rows.length;
}
// 索引路径:等值/范围查找(结果行已过滤,直接回调)
const fastPath = await this.tryIndexLookup(tableName, query);
if (fastPath !== null) {
for (const row of fastPath) {
if (!emit(row))
break;
}
return count;
}
// 全表惰性扫描(含 WHERE 过滤,不物化)
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
this.lsm.rangeScanLazy(prefix, `${prefix}\uffff`, (key, value) => {
if (count >= limit)
return;
const row = { ...value };
row[pkCol] = key.slice(prefix.length);
emit(row);
});
return count;
}
async count(tableName, query) {
this.ensureOpen();
this.ensureTable(tableName);
const rows = await this.getAllRows(tableName);
this.trimAllCaches();
if (!query?.where || Object.keys(query.where).length === 0)
return rows.length;
return rows.filter((row) => matchWhere(row, query.where)).length;
}
async clear(tableName) {
this.ensureOpen();
this.ensureTable(tableName);
const rows = await this.getAllRows(tableName);
// v0.3.3: 事务内清空走快照(删除标记),提交时生效;并写入 WAL
const walRecords = [];
for (const row of rows) {
const pkCol = this.tablePKs.get(tableName);
const key = `${tableName}:${row[pkCol]}`;
if (this.currentTxnId && this.txnSnapshot) {
this.txnSnapshot.set(key, { __txn_deleted: true });
this.mvcc.deleteVersion(tableName, String(row[pkCol]), this.currentTxnId);
}
else {
this.lsm.delete(key);
}
walRecords.push({
type: WALRecordType.DELETE,
txnId: this.currentTxnId ?? 0,
tableName,
key: String(row[pkCol]),
});
// 移除二级索引
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
}
await this.wal.appendBatch(walRecords);
this.opCounter += rows.length;
await this.checkpointManager.tick();
this.tryGC();
}
// ---- ALTER TABLEv0.4.1 ----
/**
* v0.4.1: ALTER TABLE — 结构变更真正生效于存储:
* - ADD: 持久化 schemapersistSchemas),行无需修改
* - DROP: 持久化 schema + 遍历主 LSM 重写所有行(移除该列键)+ WAL UPDATE 记录
* (通用路径 getTableSchema 返回副本,Executor 的引用修改对 Aria 无效)
*/
async alterTable(tableName, action, column) {
this.ensureOpen();
this.ensureNoDDLInTransaction('ALTER TABLE');
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
if (action === 'ADD') {
if (schema.columns[column.name]) {
throw new DatabaseError(`Column "${column.name}" already exists in table "${tableName}"`, 'COLUMN_EXISTS');
}
schema.columns[column.name] = column;
await this.persistSchemas();
return;
}
// DROP
if (!schema.columns[column.name]) {
throw new DatabaseError(`Column "${column.name}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
}
// v0.4.2-fix: 被删列是索引列 → 先清理索引 LSM(残留会导致后续同名列索引脏数据)
if (schema.columns[column.name].index || schema.columns[column.name].unique) {
const idxKey = `${tableName}:idx:${column.name}`;
const idxLsm = this.secondaryIndexes.get(idxKey);
if (idxLsm) {
try {
await idxLsm.clear();
}
catch { /* 清理失败不阻塞 */ }
this.secondaryIndexes.delete(idxKey);
}
}
delete schema.columns[column.name];
await this.persistSchemas();
// 重写主 LSM:移除所有行的该列键(find 副本无法就地删除,必须重写存储)
const prefix = `${tableName}:`;
const endKey = `${prefix}\uffff`;
await this.lsm.prefetchRange(prefix, endKey);
const entries = this.lsm.rangeScan(prefix, endKey);
const walRecords = [];
for (const [key, value] of entries) {
if (!(column.name in value))
continue;
const updated = { ...value };
delete updated[column.name];
this.lsm.put(key, updated);
// 二级索引列被删时同步清理索引
const pk = key.slice(prefix.length);
this.updateSecondaryIndexes(tableName, pk, updated, value);
walRecords.push({
type: WALRecordType.UPDATE,
txnId: this.currentTxnId ?? 0,
tableName,
key: pk,
data: updated,
});
}
await this.wal.appendBatch(walRecords);
this.opCounter += walRecords.length;
await this.checkpointManager.tick();
this.trimAllCaches();
}
async createIndex(tableName, column, unique) {
this.ensureOpen();
this.ensureNoDDLInTransaction('CREATE INDEX');
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const colDef = schema.columns[column];
if (!colDef)
throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
const idxKey = `${tableName}:idx:${column}`;
// v0.4.2-fix: 以索引 LSM 是否已建为准(schema 标记可能因重启恢复而存在,
// 但索引 LSM 未恢复 → 此前静默 return 导致索引永久缺失)
if (this.secondaryIndexes.has(idxKey))
return;
colDef.index = true;
if (unique)
colDef.unique = true;
const idxLsm = new LSM({
memtableSizeThreshold: this.config.memtableSizeThreshold,
levelSizeMultiplier: this.config.levelSizeMultiplier,
blockSize: this.config.pageSize,
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
cacheLimitBytes: this.config.bufferPoolPages * this.config.pageSize,
sstableStore: this.createSSTableStore(`idx_${tableName}_${column}`),
});
await idxLsm.init();
this.secondaryIndexes.set(idxKey, idxLsm);
// 从主 LSM 重建索引数据
const pkCol = this.tablePKs.get(tableName);
const rows = await this.getAllRows(tableName);
for (const row of rows) {
const value = row[column];
if (value !== undefined && value !== null) {
idxLsm.put(`${String(value)}:${row[pkCol]}`, { pk: row[pkCol] });
}
}
await idxLsm.flush();
await this.persistSchemas();
}
async dropIndex(tableName, column, _indexName) {
this.ensureOpen();
this.ensureNoDDLInTransaction('DROP INDEX');
this.ensureTable(tableName);
const schema = this.schemas.get(tableName);
const colDef = schema.columns[column];
if (!colDef)
throw new DatabaseError(`Column "${column}" does not exist in table "${tableName}"`, 'COLUMN_NOT_FOUND');
// 主键索引不可删除(PK 查找依赖主 LSM)
if (colDef.primaryKey) {
throw new DatabaseError(`Cannot drop primary key index on column "${column}"`, 'NOT_SUPPORTED');
}
// v0.4.1: DROP 不存在的索引应报错(此前静默成功)
if (!colDef.index && !colDef.unique && !this.secondaryIndexes.has(`${tableName}:idx:${column}`)) {
throw new DatabaseError(`Index on column "${column}" does not exist in table "${tableName}"`, 'INDEX_NOT_FOUND');
}
colDef.index = false;
colDef.unique = false;
const idxKey = `${tableName}:idx:${column}`;
const idxLsm = this.secondaryIndexes.get(idxKey);
if (idxLsm) {
await idxLsm.clear();
this.secondaryIndexes.delete(idxKey);
}
await this.persistSchemas();
}
// =======================================================================
// 事务
// =======================================================================
async beginTransaction() {
if (this.currentTxnId)
throw new DatabaseError('Transaction already in progress', 'TX_ACTIVE');
this.currentTxnId = this.mvcc.beginTransaction();
this.txnSnapshot = new Map();
await this.wal.append({
type: WALRecordType.BEGIN,
txnId: this.currentTxnId,
tableName: '',
key: '',
});
}
async commitTransaction() {
if (!this.currentTxnId)
throw new DatabaseError('No active transaction', 'TX_NONE');
// v0.4.3-fix: 先持久化 WAL COMMIT,再合并快照到 LSM —
// 崩溃在 WAL 提交后、快照合并前:恢复时 WAL 重放数据,重启一致;
// 崩溃在 WAL 提交前:commitTransaction 尚未返回,事务视为未提交(可回滚)
await this.wal.append({
type: WALRecordType.COMMIT,
txnId: this.currentTxnId,
tableName: '',
key: '',
});
await this.wal.flush();
if (this.txnSnapshot) {
for (const [key, value] of this.txnSnapshot) {
if (value.__txn_deleted) {
this.lsm.delete(key);
}
else {
this.lsm.put(key, value);
}
}
}
this.mvcc.commitTransaction(this.currentTxnId);
this.currentTxnId = null;
this.txnSnapshot = null;
}
async rollbackTransaction() {
if (!this.currentTxnId)
throw new DatabaseError('No active transaction', 'TX_NONE');
// v0.3.3: 记录事务涉及的表(用于回滚后重建索引,消除索引残留)
const affectedTables = new Set();
if (this.txnSnapshot) {
for (const key of this.txnSnapshot.keys()) {
const idx = key.indexOf(':');
if (idx > 0)
affectedTables.add(key.slice(0, idx));
}
}
this.mvcc.rollbackTransaction(this.currentTxnId);
this.txnSnapshot = null;
await this.wal.append({
type: WALRecordType.ROLLBACK,
txnId: this.currentTxnId,
tableName: '',
key: '',
});
this.currentTxnId = null;
// v0.3.3: 事务内直接写入了二级索引 LSM,回滚后全量重建受影响表的索引
for (const tableName of affectedTables) {
if (this.schemas.has(tableName)) {
await this.reindexTable(tableName);
}
}
}
async savepoint(name) {
if (!this.currentTxnId)
throw new DatabaseError('No active transaction for savepoint', 'TX_NONE');
if (this.savepoints.has(name))
throw new DatabaseError(`Savepoint "${name}" already exists`, 'SAVEPOINT_EXISTS');
// 保存当前事务快照
this.savepoints.set(name, {
txnId: this.currentTxnId,
snapshot: this.txnSnapshot ? new Map(this.txnSnapshot) : null,
});
}
async rollbackToSavepoint(name) {
const sp = this.savepoints.get(name);
if (!sp)
throw new DatabaseError(`Savepoint "${name}" not found`, 'SAVEPOINT_NOT_FOUND');
// v0.6.3-fix: 记录当前快照涉及的表(回滚后重建索引)—— 此前事务内直写
// 索引 LSMsavepoint 回滚只还原快照 → savepoint 之后的索引条目残留
const affectedTables = new Set();
if (this.txnSnapshot) {
for (const key of this.txnSnapshot.keys()) {
const idx = key.indexOf(':');
if (idx > 0)
affectedTables.add(key.slice(0, idx));
}
}
// 恢复到 savepoint 时的快照
this.txnSnapshot = sp.snapshot ? new Map(sp.snapshot) : null;
// v0.3.3: 清理该事务在 MVCC 版本链中的全部记录(快照已含正确数据,
// 版本链仅作 undo 记录,清空后 commit 时 LSM 写入与快照保持一致)
this.mvcc.discardVersions(this.currentTxnId);
// 清除此 savepoint 之后的所有 savepoint
let found = false;
for (const [k] of this.savepoints) {
if (k === name) {
found = true;
continue;
}
if (found)
this.savepoints.delete(k);
}
// v0.6.3-fix: 重建受影响表二级索引(消除 savepoint 之后的过期索引条目)
for (const tableName of affectedTables) {
if (this.schemas.has(tableName)) {
await this.reindexTable(tableName);
}
}
}
async releaseSavepoint(name) {
if (!this.savepoints.has(name))
throw new DatabaseError(`Savepoint "${name}" not found`, 'SAVEPOINT_NOT_FOUND');
this.savepoints.delete(name);
}
// ---- 在线备份 ----
async backup() {
this.ensureOpen();
const result = {};
for (const tableName of this.schemas.keys()) {
result[tableName] = await this.getAllRows(tableName);
}
return result;
}
// =======================================================================
// 内部
// =======================================================================
async getAllRows(tableName) {
const pkCol = this.tablePKs.get(tableName);
const prefix = `${tableName}:`;
// 预加载范围内涉及的 SSTable,避免 rangeScan 时缓存未命中静默丢数据
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
const rows = entries.map(([key, value]) => {
const row = { ...value };
row[pkCol] = key.slice(prefix.length);
return row;
});
// v0.3.3: 事务内合并未提交快照(update/delete/count/clear 也能看到本事务的写入)
return this.mergeTxnSnapshot(tableName, rows);
}
/**
* v0.3.3: 将事务未提交快照的变更合并到行列表(新增/更新/删除标记)。
* 幂等操作:行已是最新时不重复修改。
*/
mergeTxnSnapshot(tableName, rows) {
if (!this.currentTxnId || !this.txnSnapshot)
return rows;
const pkCol = this.tablePKs.get(tableName);
const prefix = `${tableName}:`;
for (const [key, value] of this.txnSnapshot) {
if (!key.startsWith(prefix))
continue;
const pk = key.slice(prefix.length);
const del = value.__txn_deleted;
const idx = rows.findIndex((r) => r[pkCol] === pk);
if (del) {
if (idx >= 0)
rows.splice(idx, 1);
}
else {
const row = { ...value, [pkCol]: pk };
if (idx >= 0)
rows[idx] = row;
else
rows.push(row);
}
}
return rows;
}
getPK(schema) {
for (const [name, col] of Object.entries(schema.columns)) {
if (col.primaryKey)
return name;
}
return Object.keys(schema.columns)[0];
}
validateRow(schema, row) {
const validated = {};
for (const [colName, colDef] of Object.entries(schema.columns)) {
let value = row[colName];
if (value === undefined && colDef.default !== undefined)
value = colDef.default;
if (colDef.required && (value === undefined || value === null)) {
throw new DatabaseError(`Column "${colName}" is required in table "${schema.name}"`, 'VALIDATION_ERROR');
}
if (value !== undefined && value !== null) {
this.checkType(colName, colDef.type, value, colDef);
}
if (value !== undefined)
validated[colName] = value;
}
return validated;
}
checkType(colName, type, value, colDef) {
checkFieldType('', colName, type, value, colDef);
}
// =======================================================================
// Schema 持久化
// =======================================================================
async persistSchemas() {
const data = {};
for (const [name, schema] of this.schemas) {
data[name] = schema.columns;
}
const json = JSON.stringify(data);
const buf = new TextEncoder().encode(json).buffer;
await this.backend.write('__aria_schemas', buf);
}
async loadSchemas() {
const raw = await this.backend.read('__aria_schemas');
if (!raw)
return;
try {
const json = new TextDecoder().decode(raw);
const data = JSON.parse(json);
for (const [tableName, columns] of Object.entries(data)) {
const schema = { name: tableName, columns };
this.schemas.set(tableName, schema);
this.tablePKs.set(tableName, this.getPK(schema));
}
}
catch {
// 忽略损坏的 schema 数据
}
}
// =======================================================================
// SSTableStore 构建
// =======================================================================
/**
* 创建命名空间隔离的 SSTableStore。
*
* 主 LSM 与每个二级索引 LSM 各持有独立实例:
* - 文件 key 前缀隔离(sst_ / sst_idx_${table}_${col}_
* - 元数据 key 隔离(__aria_lsm_meta / __aria_lsm_meta_${ns}
* - id 序列独立(避免 v0.2.4 共享 id 空间导致的文件互相覆盖)
*/
createSSTableStore(ns) {
const filePrefix = ns === 'main' ? 'sst_' : `sst_${ns}_`;
const META_KEY = ns === 'main' ? '__aria_lsm_meta' : `__aria_lsm_meta_${ns}`;
let seq = 0;
let seqLoaded = false;
// v0.4.5: 页面化物理存储(OPFS 后端默认启用)— SSTable 存为 4KB 页面,BufferPool 缓存
const usePages = this.isPageStorage();
const pageStore = usePages ? new PageSSTableStore(this.fileManager, this.bufferPool) : null;
const encodeText = (text) => {
return new TextEncoder().encode(text).buffer;
};
const readMetaList = async () => {
const raw = await this.backend.read(META_KEY);
if (!raw)
return [];
try {
return JSON.parse(new TextDecoder().decode(raw));
}
catch {
return [];
}
};
return {
save: async (id, data) => {
if (pageStore) {
// 页面化:切页写入 BufferPool 并逐页落盘(save 语义 = 已持久化)
await pageStore.save(id, data);
return;
}
let buf = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
// 压缩(若启用)— 加密由 EncryptedBackend 在 backend 层透明处理(v0.4.5
if (this.config.compression) {
const compressed = compressLZ4(new Uint8Array(buf));
buf = compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength);
}
await this.backend.write(`${filePrefix}${id}`, buf);
},
load: async (id) => {
// 页面化读取:meta 有 pageIds → 页面拼接;无(旧数据)→ 整 value
if (pageStore) {
const metas = await readMetaList();
const meta = metas.find((m) => m.id === id);
if (meta && meta.pageIds && meta.pageIds.length > 0) {
return pageStore.load(id, meta.pageIds, meta.totalSize);
}
}
const raw = await this.backend.read(`${filePrefix}${id}`);
if (!raw)
return null;
let buf = new Uint8Array(raw);
// 解压(若启用)— 解密由 EncryptedBackend 在 backend 层透明处理(v0.4.5
if (this.config.compression) {
// v0.4.5: 压缩流自带原始大小头,无需外部估算
buf = decompressLZ4(buf);
}
return buf;
},
delete: async (id) => {
if (pageStore) {
const metas = await readMetaList();
const meta = metas.find((m) => m.id === id);
if (meta && meta.pageIds && meta.pageIds.length > 0) {
await pageStore.delete(id, meta.pageIds);
}
}
await this.backend.delete(`${filePrefix}${id}`);
},
allocateId: async () => {
// 从本命名空间的 meta 恢复 id 序列,保证单调递增且不与其他 LSM 冲突
if (!seqLoaded) {
const metas = await readMetaList();
seq = metas.reduce((m, x) => Math.max(m, x.id), 0);
seqLoaded = true;
}
return ++seq;
},
listMeta: readMetaList,
saveMeta: async (meta) => {
const list = await readMetaList();
// v0.4.5: 页面化时把页面 ID 列表注入 metasave 后、saveMeta 前由 LSM 顺序调用)
const pageIds = pageStore?.getPageIds(meta.id);
const metaWithPages = pageIds && pageIds.length > 0 ? { ...meta, pageIds } : meta;
// 更新或添加
const idx = list.findIndex((m) => m.id === meta.id);
if (idx >= 0)
list[idx] = metaWithPages;
else
list.push(metaWithPages);
await this.backend.write(META_KEY, encodeText(JSON.stringify(list)));
},
deleteMeta: async (id) => {
const list = await readMetaList();
const filtered = list.filter((m) => m.id !== id);
await this.backend.write(META_KEY, encodeText(JSON.stringify(filtered)));
},
};
}
/** v0.4.5: 是否启用页面化物理存储(默认 OPFS / KVStore 后端启用,显式配置可覆盖) */
isPageStorage() {
if (this.config.pageStorage === true)
return true;
if (this.config.pageStorage === false)
return false;
// v0.6.1: kv 后端同样页面化 — 整 value SSTable4MB)超过 1MB 页面缓存时
// 每次读取全量重载;拆 4KB 页面后缓存按页命中,大数据量读放大消除
return this.config.storageBackend === 'opfs' || this.config.storageBackend === 'kv';
}
// =======================================================================
// WAL 恢复
// =======================================================================
applyWALRecord(record) {
switch (record.type) {
case WALRecordType.INSERT:
case WALRecordType.UPDATE:
if (record.data) {
this.lsm.put(`${record.tableName}:${record.key}`, record.data);
}
break;
case WALRecordType.DELETE:
this.lsm.delete(`${record.tableName}:${record.key}`);
break;
case WALRecordType.CREATE_TABLE:
if (record.data?.schema) {
try {
const s = JSON.parse(record.data.schema);
if (!this.schemas.has(s.name)) {
this.schemas.set(s.name, s);
this.tablePKs.set(s.name, this.getPK(s));
}
}
catch { /* skip */ }
}
break;
case WALRecordType.COMMIT:
case WALRecordType.ROLLBACK:
case WALRecordType.BEGIN:
break;
}
}
/**
* v0.3.3: DROP_TABLE 恢复 — 删除 schema 并清除主 LSM 中该表的所有残留数据。
*
* 此前 DROP_TABLE 在恢复时被忽略,而 CREATE_TABLE 回放会重建 schema
* 导致崩溃后"已删除的表和数据复活"(实证 P0 bug)。
*/
async applyDropTableRecovery(tableName) {
if (!tableName)
return;
// v0.4.2-fix: 清理该表二级索引(崩溃恢复路径同样不留孤儿索引)
await this.cleanupTableIndexes(tableName);
this.schemas.delete(tableName);
this.tablePKs.delete(tableName);
// 清除主 LSM 中该表前缀的所有数据(含 SSTable 中的旧数据)
const prefix = `${tableName}:`;
const endKey = `${prefix}\uffff`;
await this.lsm.prefetchRange(prefix, endKey);
const entries = this.lsm.rangeScan(prefix, endKey);
for (const [key] of entries) {
this.lsm.delete(key);
}
}
// =======================================================================
// 二级索引
// =======================================================================
/** v0.6.2: 表中有 unique 约束且索引 LSM 已建的列(唯一性检查范围) */
uniqueColumns(tableName, schema) {
const cols = [];
for (const [colName, colDef] of Object.entries(schema.columns)) {
if (!colDef.unique)
continue;
if (this.secondaryIndexes.has(`${tableName}:idx:${colName}`))
cols.push(colName);
}
return cols;
}
/**
* v0.6.2: 同步唯一性检查(须在批次级 prefetchPrefixRanges 之后调用,循环内无 await)。
* 索引不含 null 条目(null 值不受唯一约束,与 MemoryEngine 语义一致)。
* @param currentPk 当前行主键(更新路径用于排除自身旧索引条目;插入路径无自身条目)
*/
checkUniqueSync(tableName, uniqueCols, row, currentPk) {
for (const colName of uniqueCols) {
const val = row[colName];
if (val === undefined || val === null)
continue;
const idxLsm = this.secondaryIndexes.get(`${tableName}:idx:${colName}`);
if (!idxLsm)
continue;
const prefix = `${String(val)}:`;
const entries = idxLsm.rangeScan(prefix, `${prefix}\uffff`);
for (const [, entry] of entries) {
const pk = entry.pk;
if (pk !== undefined && pk !== currentPk) {
throw new DatabaseError(`Unique constraint violation on column "${colName}" in table "${tableName}"`, 'UNIQUE_VIOLATION');
}
}
}
}
/** 更新行的二级索引条目 */
updateSecondaryIndexes(tableName, pkValue, newRow, oldRow) {
const schema = this.schemas.get(tableName);
if (!schema)
return;
for (const [colName, colDef] of Object.entries(schema.columns)) {
// v0.3.3: 主键列不建冗余二级索引(主 LSM 即 PK 索引)
if (!colDef.index && !colDef.unique)
continue;
const idxKey = `${tableName}:idx:${colName}`;
const idxLsm = this.secondaryIndexes.get(idxKey);
if (!idxLsm)
continue;
// 删除旧值
if (oldRow) {
const oldVal = oldRow[colName];
if (oldVal !== undefined && oldVal !== null) {
idxLsm.delete(`${String(oldVal)}:${pkValue}`);
}
}
// 插入新值
if (newRow) {
const newVal = newRow[colName];
if (newVal !== undefined && newVal !== null) {
idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue });
}
}
}
}
/** 通过二级索引快速查找 */
async tryIndexLookup(tableName, query) {
if (!query.where)
return null;
const schema = this.schemas.get(tableName);
if (!schema)
return null;
const pkCol = this.tablePKs.get(tableName);
for (const [col, condition] of Object.entries(query.where)) {
// 跳过 $and/$or/$not 逻辑组合
if (col === '$and' || col === '$or' || col === '$not')
continue;
const colDef = schema.columns[col];
const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey);
if (!hasIndex && col !== pkCol)
continue;
// PK 等值 → 主 LSM 精确查找
if (col === pkCol) {
if (typeof condition !== 'object' || condition === null) {
const key = `${tableName}:${condition}`;
await this.lsm.prefetchKeys([key]);
const value = this.lsm.get(key);
return value ? [{ ...value, [pkCol]: condition }] : [];
}
const cond = condition;
if ('$eq' in cond) {
const key = `${tableName}:${cond.$eq}`;
await this.lsm.prefetchKeys([key]);
const value = this.lsm.get(key);
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
}
// v0.3.3: PK $in → 主 LSM 多次精确查找(替代冗余 PK 二级索引)
if ('$in' in cond && Array.isArray(cond.$in)) {
const keys = cond.$in.map((v) => `${tableName}:${v}`);
await this.lsm.prefetchKeys(keys);
const rows = [];
const seen = new Set(); // v0.4.1: IN 子查询可能含重复值,按 pk 去重
for (const v of cond.$in) {
const pk = String(v);
if (seen.has(pk))
continue;
const value = this.lsm.get(`${tableName}:${pk}`);
if (value) {
seen.add(pk);
rows.push({ ...value, [pkCol]: pk });
}
}
return rows;
}
// v0.3.3: PK 范围查询 → 主 LSM 前缀扫描 + 条件过滤(修复字符串算术 bug)
if ('$gt' in cond || '$gte' in cond || '$lt' in cond || '$lte' in cond) {
const prefix = `${tableName}:`;
await this.lsm.prefetchRange(prefix, `${prefix}\uffff`);
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
const rows = [];
for (const [key, value] of entries) {
const candidate = { ...value, [pkCol]: key.slice(prefix.length) };
if (matchWhere(candidate, { [pkCol]: condition }))
rows.push(candidate);
}
return rows;
}
}
// 二级索引查找
const idxKey = `${tableName}:idx:${col}`;
const idxLsm = this.secondaryIndexes.get(idxKey);
if (!idxLsm)
continue;
// $eq → 精确查找
if (typeof condition !== 'object' || condition === null) {
// v0.6.2-fix(P1): IS NULL(条件为 null)不走索引 —— 索引不含 null 条目,
// String(null)="null" 查找返回空并短路全表 → 索引列 IS NULL 恒空
if (condition === null)
continue;
return this.indexScanToRows(tableName, pkCol, idxLsm, String(condition), String(condition));
}
const c = condition;
if ('$eq' in c) {
// v0.6.2-fix(P1): 同上,$eq: nullIS NULL)不走索引
if (c.$eq === null)
continue;
const v = String(c.$eq);
return this.indexScanToRows(tableName, pkCol, idxLsm, v, v);
}
// $in → 多次精确查找
if ('$in' in c && Array.isArray(c.$in)) {
// v0.6.2-fix(P1): IN 列表含 null 不走索引(索引不含 null 条目,会漏匹配 null 行)
if (c.$in.some((v) => v === null))
continue;
const results = [];
const seenPks = new Set(); // v0.4.1: IN 值可能重复,按 pk 去重
for (const val of c.$in) {
const rows = await this.indexScanToRows(tableName, pkCol, idxLsm, String(val), String(val));
for (const row of rows) {
const pk = String(row[pkCol]);
if (!seenPks.has(pk)) {
seenPks.add(pk);
results.push(row);
}
}
}
return results;
}
// $gt / $gte / $lt / $lte → 范围扫描
if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) {
// v0.6.2-fix(P1): 此前用 Number(v)±1 构造边界 key —— 小数数值
// $gt:2 → "3:",漏 2.5)与字符串("NaN:" 前缀错位,数字/大写开头值被漏)
// 静默丢数据。改为全索引扫描 + 行级 matchWhere 过滤(与主键范围路径同方案),
// 边界语义与 where-matcher 完全一致。
const rows = await this.indexScanToRows(tableName, pkCol, idxLsm, '', '\uffff');
return rows.filter((row) => matchWhere(row, { [col]: condition }));
}
}
return null;
}
/** 从索引扫描结果恢复完整行 */
async indexScanToRows(tableName, pkCol, idxLsm, startKey, endKey) {
// 使用前缀扫描:endKey 需要包含 \uffff 以匹配所有带后缀的 key
const actualEndKey = endKey.includes('\uffff') ? endKey : `${endKey}\uffff`;
// 预加载索引 LSM 与主 LSM 涉及的 SSTable
await idxLsm.prefetchRange(startKey, actualEndKey);
const entries = idxLsm.rangeScan(startKey, actualEndKey);
const pks = [];
for (const [, idxEntry] of entries) {
const pk = idxEntry.pk;
if (pk)
pks.push(pk);
}
await this.lsm.prefetchKeys(pks.map((pk) => `${tableName}:${pk}`));
const rows = [];
for (const pk of pks) {
const row = this.lsm.get(`${tableName}:${pk}`);
if (row)
rows.push({ ...row, [pkCol]: pk });
}
return rows;
}
// =======================================================================
// 辅助
// =======================================================================
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
tryGC() {
this.gcCounter++;
if (this.gcCounter >= 10) {
this.mvcc.gc(100);
this.gcCounter = 0;
}
}
/** 回收主 LSM 与所有二级索引 LSM 的临时缓存超限 */
trimAllCaches() {
this.lsm.trimCache();
for (const idxLsm of this.secondaryIndexes.values()) {
idxLsm.trimCache();
}
}
/** 检查内存预算,超出时强制 flush + GC */
checkMemoryBudget() {
const maxBytes = this.config.maxMemoryMB * 1024 * 1024;
const used = this.lsm.getEstimatedMemory();
if (used > maxBytes) {
this.lsm.flush().catch(() => { });
this.mvcc.gc(50);
}
}
/** 估算 WAL 大小(字节) */
getWALEstimatedSize() {
return this.wal.getBufferedCount() * 200; // 粗略估算每条 ~200B
}
/**
* ANALYZE: 收集表统计信息
* 返回行数、平均行大小、索引深度等
*/
async analyzeTable(tableName) {
this.ensureOpen();
this.ensureTable(tableName);
const rows = await this.getAllRows(tableName);
const stats = {
table: tableName,
rowCount: rows.length,
avgRowSize: rows.length > 0
? Math.round(rows.reduce((s, r) => s + JSON.stringify(r).length, 0) / rows.length)
: 0,
indexDepth: this.lsm.getStats().levelCounts.filter((c) => c > 0).length,
sstableCount: this.lsm.getStats().sstableCount,
memtableSize: this.lsm.getStats().memtableSize,
estimatedMemory: this.lsm.getEstimatedMemory(),
};
// 列基数统计
const schema = this.schemas.get(tableName);
if (schema && rows.length > 0) {
const columnStats = {};
for (const colName of Object.keys(schema.columns)) {
const values = new Set(rows.map((r) => String(r[colName])));
columnStats[colName] = { distinctValues: values.size };
}
stats.columnStats = columnStats;
}
return stats;
}
/**
* REINDEX: 重建指定表的所有二级索引
*/
async reindexTable(tableName) {
this.ensureOpen();
this.ensureTable(tableName);
return this.reindexTableInternal(tableName);
}
/** v0.4.2-fix: 重建索引内部实现(不校验 opened,供 open 恢复流程调用) */
async reindexTableInternal(tableName) {
const schema = this.schemas.get(tableName);
if (!schema)
return 0;
let rebuiltCount = 0;
for (const [colName, colDef] of Object.entries(schema.columns)) {
// v0.3.3: 主键列不建冗余二级索引(主 LSM 即 PK 索引)
if (!colDef.index && !colDef.unique)
continue;
const idxKey = `${tableName}:idx:${colName}`;
const idxLsm = this.secondaryIndexes.get(idxKey);
if (!idxLsm)
continue;
// 清空旧索引
await idxLsm.clear();
rebuiltCount++;
// 从主 LSM 重建索引
const rows = await this.getAllRows(tableName);
for (const row of rows) {
const val = row[colName];
if (val !== undefined && val !== null) {
idxLsm.put(`${String(val)}:${row[this.tablePKs.get(tableName)]}`, { pk: row[this.tablePKs.get(tableName)] });
}
}
}
return rebuiltCount;
}
/**
* VACUUM: 压缩 LSM + 清理碎片
*/
async vacuum() {
this.ensureOpen();
// 强制 flush memtable
await this.lsm.flush();
// 压缩各层级
for (let level = 0; level < 6; level++) {
if (this.lsm.getStats().levelCounts[level] >= 2) {
await this.lsm.compactLevel(level);
}
}
// GC MVCC 版本(保留最新 10 个)
const beforeGC = this.mvcc.getGlobalLSN();
this.mvcc.gc(10);
return { compactedLevels: 6, gcVersions: beforeGC };
}
ensureOpen() {
if (!this.opened)
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
}
/** v0.4.2-fix: Aria 事务中 DDL 显式拒绝(结构变更无法通过行快照回滚) */
ensureNoDDLInTransaction(op) {
if (this.currentTxnId) {
throw new DatabaseError(`${op} is not supported inside a transaction (AriaEngine DDL is not transactional)`, 'NOT_SUPPORTED');
}
}
ensureTable(tableName) {
if (!this.schemas.has(tableName)) {
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
}
}
}
/**
* metona-sqlark Hybrid Engine — 内存 + 磁盘混合存储引擎
* @module hybrid/index
*
* 采用 write-through 策略:
* - 所有写操作同时写入内存和磁盘
* - 所有读操作直接从内存返回
* - 数据库打开时从磁盘加载数据到内存
*/
// ---------------------------------------------------------------------------
// HybridEngine
// ---------------------------------------------------------------------------
class HybridEngine {
constructor(diskEngine = 'opfs') {
this.name = 'hybrid';
this.dbName = '';
this.version = 1;
this.memoryEngine = new MemoryEngine();
this.diskEngineType = diskEngine;
// v0.6.0: 磁盘层统一为自研 KVStoreEngine
this.diskEngine = new KVStoreEngine();
}
// ---- 生命周期 ----
async open(dbName, version) {
this.dbName = dbName;
this.version = version;
// 先打开磁盘引擎
await this.diskEngine.open(dbName, version);
// 再打开内存引擎
await this.memoryEngine.open(dbName, version);
// 从磁盘加载现存表
await this.reloadMemoryFromDisk();
}
/**
* 从磁盘重载内存缓存(v0.3.2:多标签页同步)。
* 其他标签页写入磁盘后调用,使本标签页读到最新数据。
*/
async reloadMemoryFromDisk() {
await this.memoryEngine.close();
await this.memoryEngine.open(this.dbName, this.version);
// v0.6.0: KVStoreEngine 读内存 → 先重载磁盘最新数据
const disk = this.diskEngine;
if (typeof disk.reload === 'function') {
await disk.reload();
}
const tableNames = await this.diskEngine.getTableNames();
for (const tableName of tableNames) {
const schema = await this.diskEngine.getTableSchema(tableName);
if (!schema)
continue;
// 在内存中创建表
await this.memoryEngine.createTable(schema);
// 从磁盘加载数据到内存
const rows = await this.diskEngine.find(tableName, { table: tableName });
if (rows.length > 0) {
try {
await this.memoryEngine.insert(tableName, rows);
}
catch (e) {
// eslint-disable-next-line no-console
console.warn(`[metona-sqlark] Failed to load table "${tableName}" data from disk:`, e);
}
}
}
}
async close() {
await this.memoryEngine.close();
await this.diskEngine.close();
}
isOpen() {
return this.memoryEngine.isOpen() && this.diskEngine.isOpen();
}
// ---- v0.4.2-fix: 自愈 / 重置 / 元数据(委托双引擎) ----
/** 自愈:修复磁盘引擎后重载内存缓存 */
async repair() {
if (typeof this.diskEngine.repair === 'function') {
await this.diskEngine.repair();
}
await this.reloadMemoryFromDisk();
}
/** 清空全部数据与表结构 */
async clearAll() {
if (typeof this.diskEngine.clearAll === 'function') {
await this.diskEngine.clearAll();
}
else {
const names = await this.diskEngine.getTableNames();
for (const name of names) {
await this.diskEngine.dropTable(name);
}
}
if (typeof this.memoryEngine.clearAll === 'function') {
await this.memoryEngine.clearAll();
}
else {
const names = await this.memoryEngine.getTableNames();
for (const name of names) {
await this.memoryEngine.dropTable(name);
}
}
}
async getMeta(key) {
if (typeof this.diskEngine.getMeta === 'function') {
return this.diskEngine.getMeta(key);
}
return null;
}
async setMeta(key, value) {
if (typeof this.diskEngine.setMeta === 'function') {
await this.diskEngine.setMeta(key, value);
}
}
// ---- 表管理 ----
async createTable(schema) {
await this.memoryEngine.createTable(schema);
try {
await this.diskEngine.createTable(schema);
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
}
async dropTable(tableName) {
await this.memoryEngine.dropTable(tableName);
try {
await this.diskEngine.dropTable(tableName);
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
}
async hasTable(tableName) {
return this.memoryEngine.hasTable(tableName);
}
async getTableNames() {
return this.memoryEngine.getTableNames();
}
async getTableSchema(tableName) {
return this.memoryEngine.getTableSchema(tableName);
}
/** v0.4.2-fix: 引擎级 ALTER TABLE — 双引擎同步(磁盘持久化 + 内存引用) */
async alterTable(tableName, action, column) {
await this.memoryEngine.alterTable(tableName, action, column);
try {
if (typeof this.diskEngine.alterTable === 'function') {
await this.diskEngine.alterTable(tableName, action, column);
}
else {
// 磁盘引擎无引擎级实现 → 从磁盘重建内存 schema(disk 引擎 schema 以自身为准)
const schema = await this.diskEngine.getTableSchema(tableName);
if (schema && action === 'DROP')
delete schema.columns[column.name];
}
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
}
// ---- CRUDwrite-through 策略) ----
/**
* v0.7.2: 磁盘写失败补偿 — 内存已先行写入、磁盘失败 → 内存与磁盘不一致
* (重启后数据丢失且调用方已收到错误)。从磁盘重载内存对齐真实状态
* (内存=磁盘),再重新抛出原始错误。事务路径由双引擎快照回滚保证,
* 无需此补偿。
*/
async recoverMemoryAfterDiskError(error) {
try {
await this.reloadMemoryFromDisk();
}
catch {
// 磁盘本身不可用(错误根源)时重载可能失败:错误已抛给调用方,
// 内存保持失败前状态,repair()/重试可恢复
// eslint-disable-next-line no-console
console.warn('[metona-sqlark] Hybrid: failed to reload memory after disk write error');
}
throw error;
}
async insert(tableName, rows) {
const pks = await this.memoryEngine.insert(tableName, rows);
// write-through: 同步写入磁盘
try {
await this.diskEngine.insert(tableName, rows);
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
return pks;
}
async find(tableName, query) {
// 直接从内存读取
return this.memoryEngine.find(tableName, query);
}
/** v0.4.0: 流式查询(内存引擎逐行回调) */
async findStream(tableName, query, onRow) {
return this.memoryEngine.findStream(tableName, query, onRow);
}
async update(tableName, query, updates) {
const count = await this.memoryEngine.update(tableName, query, updates);
// write-through: 同步更新磁盘
try {
await this.diskEngine.update(tableName, query, updates);
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
return count;
}
async delete(tableName, query) {
const count = await this.memoryEngine.delete(tableName, query);
// write-through: 同步删除磁盘
try {
await this.diskEngine.delete(tableName, query);
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
return count;
}
async count(tableName, query) {
return this.memoryEngine.count(tableName, query);
}
async clear(tableName) {
await this.memoryEngine.clear(tableName);
try {
await this.diskEngine.clear(tableName);
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
}
// ---- 动态索引(v0.3.0 ----
async createIndex(tableName, column, unique) {
await this.memoryEngine.createIndex(tableName, column, unique);
try {
if (typeof this.diskEngine.createIndex === 'function') {
await this.diskEngine.createIndex(tableName, column, unique);
}
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
}
async dropIndex(tableName, column, indexName) {
await this.memoryEngine.dropIndex(tableName, column, indexName);
try {
if (typeof this.diskEngine.dropIndex === 'function') {
await this.diskEngine.dropIndex(tableName, column, indexName);
}
}
catch (error) {
await this.recoverMemoryAfterDiskError(error);
}
}
// ---- 事务 ----
async beginTransaction() {
await this.memoryEngine.beginTransaction();
await this.diskEngine.beginTransaction();
}
async commitTransaction() {
// 先写磁盘,保证持久化优先;磁盘失败则回滚内存
await this.diskEngine.commitTransaction();
try {
await this.memoryEngine.commitTransaction();
}
catch (error) {
// v0.4.2-fix: 磁盘已提交无法回滚(此前调 diskEngine.rollbackTransaction()
// 会抛 TX_NONE 掩盖原错误)。如实上报内存提交失败,磁盘数据保持已提交状态。
throw new DatabaseError('Hybrid commit failed: memory engine error after disk commit (disk data is committed)', 'TX_COMMIT_ERROR', error);
}
}
async rollbackTransaction() {
await this.memoryEngine.rollbackTransaction();
await this.diskEngine.rollbackTransaction();
}
// ---- 引擎信息 ----
/** 获取磁盘引擎类型 */
getDiskEngineType() {
return this.diskEngineType;
}
/** 获取内存引擎(供内部使用) */
getMemoryEngine() {
return this.memoryEngine;
}
}
/**
* metona-sqlark Query Builder — 链式查询构建器
* @module query/builder
*
* 链式调用 → 构建 AST → 执行引擎操作。
* 支持 JOIN(需要 Executor)。
*/
// ---------------------------------------------------------------------------
// SelectQueryBuilder
// ---------------------------------------------------------------------------
class SelectQueryBuilder {
constructor(engine, tableName, _columns = ['*'], executor) {
this.engine = engine;
this.tableName = tableName;
this._columns = _columns;
this._where = {};
this._orderBy = [];
this._joins = [];
this._executor = executor;
}
/** 主表别名 */
as(alias) {
this._alias = alias;
return this;
}
/** INNER JOIN */
innerJoin(table, on, alias) {
return this._addJoin('INNER', table, on, alias);
}
/** LEFT JOIN */
leftJoin(table, on, alias) {
return this._addJoin('LEFT', table, on, alias);
}
/** RIGHT JOIN */
rightJoin(table, on, alias) {
return this._addJoin('RIGHT', table, on, alias);
}
/** CROSS JOIN */
crossJoin(table, alias) {
return this._addJoin('CROSS', table, {}, alias);
}
/** 通用 JOIN */
join(table, on, alias) {
return this._addJoin('INNER', table, on, alias);
}
_addJoin(type, table, on, alias) {
this._joins.push({ type, table, on, alias });
return this;
}
/** 添加过滤条件 */
where(condition) {
this._where = { ...this._where, ...condition };
return this;
}
/** 排序 */
orderBy(column, direction = 'asc') {
this._orderBy.push({ column, direction });
return this;
}
/** 限制返回条数 */
limit(n) {
this._limit = n;
return this;
}
/** 偏移量 */
offset(n) {
this._offset = n;
return this;
}
/** 执行查询 */
async execute() {
// 有 JOIN → 通过 Executor 执行
if (this._joins.length > 0 && this._executor) {
const ast = this.toAST();
return this._executor.execute(ast);
}
// 无 JOIN → 直接调用引擎
return this.engine.find(this.tableName, {
table: this.tableName,
columns: this._columns,
where: this._where,
orderBy: this._orderBy.length > 0 ? this._orderBy : undefined,
limit: this._limit,
offset: this._offset,
});
}
/** 获取 AST */
toAST() {
return {
type: 'SELECT',
columns: this._columns,
from: this.tableName,
alias: this._alias,
joins: this._joins.length > 0 ? [...this._joins] : undefined,
where: this._where,
orderBy: this._orderBy.length > 0 ? this._orderBy : undefined,
limit: this._limit,
offset: this._offset,
};
}
}
// ---------------------------------------------------------------------------
// UpdateQueryBuilder
// ---------------------------------------------------------------------------
class UpdateQueryBuilder {
constructor(engine, tableName, _updates, onWrite, onHooks) {
this.engine = engine;
this.tableName = tableName;
this._updates = _updates;
this._where = {};
this.onWrite = onWrite;
this.onHooks = onHooks;
}
where(condition) {
this._where = { ...this._where, ...condition };
return this;
}
async execute() {
const query = { table: this.tableName, where: this._where };
await this.onHooks?.('beforeUpdate', [query, this._updates]);
const count = await this.engine.update(this.tableName, query, this._updates);
this.onWrite?.(this.tableName);
await this.onHooks?.('afterUpdate', [query, this._updates, count]);
return count;
}
toAST() {
return { type: 'UPDATE', table: this.tableName, sets: this._updates, where: this._where };
}
}
// ---------------------------------------------------------------------------
// DeleteQueryBuilder
// ---------------------------------------------------------------------------
class DeleteQueryBuilder {
constructor(engine, tableName, onWrite, onHooks) {
this.engine = engine;
this.tableName = tableName;
this._where = {};
this.onWrite = onWrite;
this.onHooks = onHooks;
}
where(condition) {
this._where = { ...this._where, ...condition };
return this;
}
async execute() {
const query = { table: this.tableName, where: this._where };
await this.onHooks?.('beforeDelete', [query]);
const count = await this.engine.delete(this.tableName, query);
this.onWrite?.(this.tableName);
await this.onHooks?.('afterDelete', [query, count]);
return count;
}
toAST() {
return { type: 'DELETE', from: this.tableName, where: this._where };
}
}
/**
* metona-sqlark Table — 表操作 API
* @module table/table
*/
// ---------------------------------------------------------------------------
// Table
// ---------------------------------------------------------------------------
class Table {
constructor(engine, tableName, executor, onWrite, onHooks) {
this.schema = null;
this.engine = engine;
this.name = tableName;
this.executor = executor;
this.onWrite = onWrite;
this.onHooks = onHooks;
}
// ---- Schema ----
async getSchema() {
if (!this.schema) {
const s = await this.engine.getTableSchema(this.name);
if (!s)
throw new DatabaseError(`Table "${this.name}" does not exist`, 'TABLE_NOT_FOUND');
this.schema = s;
}
return this.schema;
}
// ---- 插入 ----
async insert(row) {
// beforeInsert 参数约定:rows[](单行时也是数组)
await this.onHooks?.('beforeInsert', [[row]]);
const pks = await this.engine.insert(this.name, [row]);
this.onWrite?.(this.name);
await this.onHooks?.('afterInsert', [[row], pks]);
return pks[0];
}
async insertMany(rows) {
await this.onHooks?.('beforeInsert', [rows]);
const pks = await this.engine.insert(this.name, rows);
this.onWrite?.(this.name);
await this.onHooks?.('afterInsert', [rows, pks]);
return pks;
}
// ---- 查询 ----
select(columns = ['*']) {
return new SelectQueryBuilder(this.engine, this.name, columns, this.executor);
}
/** v0.4.0: 流式查询 — 逐行回调,不物化全部结果 */
async stream(onRow, query = {}) {
if (typeof this.engine.findStream !== 'function') {
const rows = await this.engine.find(this.name, {
table: this.name,
where: query.where,
limit: query.limit,
offset: query.offset,
columns: query.columns,
});
for (const row of rows)
onRow(row);
return rows.length;
}
return this.engine.findStream(this.name, {
table: this.name,
where: query.where,
limit: query.limit,
offset: query.offset,
columns: query.columns,
}, onRow);
}
// ---- 更新 ----
update(updates) {
return new UpdateQueryBuilder(this.engine, this.name, updates, this.onWrite, this.onHooks);
}
// ---- 删除 ----
delete() {
return new DeleteQueryBuilder(this.engine, this.name, this.onWrite, this.onHooks);
}
// ---- 聚合 ----
async count(where) {
return this.engine.count(this.name, where ? { table: this.name, where } : undefined);
}
// ---- 管理 ----
async clear() {
await this.engine.clear(this.name);
this.onWrite?.(this.name);
}
async drop() {
await this.engine.dropTable(this.name);
this.onWrite?.(this.name);
}
}
/**
* metona-sqlark Query Compiler — AST → 查询计划
* @module query/compiler
*
* 将 AST 语句编译为引擎可执行的 QueryPlan。
* v0.0.1: 简单直接映射,未来可加入索引选择、过滤下推等优化。
*/
// ---------------------------------------------------------------------------
// 编译 AST → QueryPlan
// ---------------------------------------------------------------------------
/**
* 编译 SELECT / DELETE / UPDATE 语句为 QueryPlan。
* INSERT 和 DDL 语句不需要 QueryPlan。
*/
function compileStatement(stmt) {
switch (stmt.type) {
case 'SELECT':
return compileSelect(stmt);
case 'DELETE':
return compileDelete(stmt);
case 'UPDATE':
return compileUpdate(stmt);
default:
throw new DatabaseError(`Cannot compile statement type "${stmt.type}" to QueryPlan`, 'COMPILE_ERROR');
}
}
function compileSelect(stmt) {
return {
table: stmt.from,
columns: stmt.columns,
where: stmt.where,
orderBy: stmt.orderBy?.length ? stmt.orderBy : undefined,
limit: stmt.limit,
offset: stmt.offset,
};
}
function compileDelete(stmt) {
return {
table: stmt.from,
where: stmt.where,
};
}
function compileUpdate(stmt) {
return {
table: stmt.table,
where: stmt.where,
};
}
/**
* metona-sqlark SQL Token Types — 词法单元定义
* @module sql/tokens
*/
// ---------------------------------------------------------------------------
// Token 类型枚举
// ---------------------------------------------------------------------------
var TokenType;
(function (TokenType) {
// 关键字
TokenType["SELECT"] = "SELECT";
TokenType["FROM"] = "FROM";
TokenType["WHERE"] = "WHERE";
TokenType["INSERT"] = "INSERT";
TokenType["INTO"] = "INTO";
TokenType["VALUES"] = "VALUES";
TokenType["UPDATE"] = "UPDATE";
TokenType["SET"] = "SET";
TokenType["DELETE"] = "DELETE";
TokenType["CREATE"] = "CREATE";
TokenType["TABLE"] = "TABLE";
TokenType["DROP"] = "DROP";
TokenType["ORDER"] = "ORDER";
TokenType["BY"] = "BY";
TokenType["ASC"] = "ASC";
TokenType["DESC"] = "DESC";
TokenType["LIMIT"] = "LIMIT";
TokenType["OFFSET"] = "OFFSET";
TokenType["AND"] = "AND";
TokenType["OR"] = "OR";
TokenType["NOT"] = "NOT";
TokenType["LIKE"] = "LIKE";
TokenType["IN"] = "IN";
TokenType["PRIMARY"] = "PRIMARY";
TokenType["KEY"] = "KEY";
TokenType["UNIQUE"] = "UNIQUE";
TokenType["DEFAULT"] = "DEFAULT";
TokenType["NULL"] = "NULL";
TokenType["TRUE"] = "TRUE";
TokenType["REFERENCES"] = "REFERENCES";
TokenType["CASCADE"] = "CASCADE";
TokenType["BETWEEN"] = "BETWEEN";
TokenType["IF"] = "IF";
TokenType["EXISTS"] = "EXISTS";
TokenType["FALSE"] = "FALSE";
TokenType["ALTER"] = "ALTER";
TokenType["ADD"] = "ADD";
TokenType["TRUNCATE"] = "TRUNCATE";
// JOIN 相关
TokenType["INNER"] = "INNER";
TokenType["LEFT"] = "LEFT";
TokenType["RIGHT"] = "RIGHT";
TokenType["CROSS"] = "CROSS";
TokenType["JOIN"] = "JOIN";
TokenType["ON"] = "ON";
TokenType["AS"] = "AS";
TokenType["OUTER"] = "OUTER";
// 聚合
TokenType["GROUP"] = "GROUP";
TokenType["HAVING"] = "HAVING";
TokenType["COUNT"] = "COUNT";
TokenType["SUM"] = "SUM";
TokenType["AVG"] = "AVG";
TokenType["MIN"] = "MIN";
TokenType["MAX"] = "MAX";
TokenType["DISTINCT"] = "DISTINCT";
// v0.3.0: 事务 / UNION / EXISTS / 动态索引
TokenType["BEGIN"] = "BEGIN";
TokenType["COMMIT"] = "COMMIT";
TokenType["ROLLBACK"] = "ROLLBACK";
TokenType["UNION"] = "UNION";
TokenType["ALL"] = "ALL";
TokenType["INDEX"] = "INDEX";
// v0.3.1: CASE WHEN 表达式
TokenType["CASE"] = "CASE";
TokenType["WHEN"] = "WHEN";
TokenType["THEN"] = "THEN";
TokenType["ELSE"] = "ELSE";
TokenType["END"] = "END";
// v0.5.1: 维护语句(EXPLAIN / ANALYZE / REINDEX / VACUUM / SAVEPOINT
TokenType["EXPLAIN"] = "EXPLAIN";
TokenType["ANALYZE"] = "ANALYZE";
TokenType["REINDEX"] = "REINDEX";
TokenType["VACUUM"] = "VACUUM";
TokenType["SAVEPOINT"] = "SAVEPOINT";
TokenType["RELEASE"] = "RELEASE";
TokenType["TO"] = "TO";
// 标识符 & 字面量
TokenType["IDENTIFIER"] = "IDENTIFIER";
TokenType["STRING"] = "STRING";
TokenType["NUMBER"] = "NUMBER";
// 运算符 & 分隔符
TokenType["COMMA"] = "COMMA";
TokenType["LPAREN"] = "LPAREN";
TokenType["RPAREN"] = "RPAREN";
TokenType["SEMICOLON"] = "SEMICOLON";
TokenType["EQ"] = "EQ";
TokenType["NEQ"] = "NEQ";
TokenType["GT"] = "GT";
TokenType["GTE"] = "GTE";
TokenType["LT"] = "LT";
TokenType["LTE"] = "LTE";
TokenType["STAR"] = "STAR";
TokenType["DOT"] = "DOT";
// 特殊
TokenType["EOF"] = "EOF";
TokenType["ILLEGAL"] = "ILLEGAL";
})(TokenType || (TokenType = {}));
// ---------------------------------------------------------------------------
// 关键字映射
// ---------------------------------------------------------------------------
const KEYWORDS = {
'SELECT': TokenType.SELECT,
'FROM': TokenType.FROM,
'WHERE': TokenType.WHERE,
'INSERT': TokenType.INSERT,
'INTO': TokenType.INTO,
'VALUES': TokenType.VALUES,
'UPDATE': TokenType.UPDATE,
'SET': TokenType.SET,
'DELETE': TokenType.DELETE,
'CREATE': TokenType.CREATE,
'TABLE': TokenType.TABLE,
'DROP': TokenType.DROP,
'ORDER': TokenType.ORDER,
'BY': TokenType.BY,
'ASC': TokenType.ASC,
'DESC': TokenType.DESC,
'LIMIT': TokenType.LIMIT,
'OFFSET': TokenType.OFFSET,
'AND': TokenType.AND,
'OR': TokenType.OR,
'NOT': TokenType.NOT,
'LIKE': TokenType.LIKE,
'IN': TokenType.IN,
'PRIMARY': TokenType.PRIMARY,
'KEY': TokenType.KEY,
'UNIQUE': TokenType.UNIQUE,
'DEFAULT': TokenType.DEFAULT,
'NULL': TokenType.NULL,
'TRUE': TokenType.TRUE,
'FALSE': TokenType.FALSE,
'REFERENCES': TokenType.REFERENCES,
'CASCADE': TokenType.CASCADE,
'BETWEEN': TokenType.BETWEEN,
'IF': TokenType.IF,
'EXISTS': TokenType.EXISTS,
'ALTER': TokenType.ALTER,
'ADD': TokenType.ADD,
'TRUNCATE': TokenType.TRUNCATE,
// JOIN
'INNER': TokenType.INNER,
'LEFT': TokenType.LEFT,
'RIGHT': TokenType.RIGHT,
'CROSS': TokenType.CROSS,
'JOIN': TokenType.JOIN,
'ON': TokenType.ON,
'AS': TokenType.AS,
'OUTER': TokenType.OUTER,
// 聚合
'GROUP': TokenType.GROUP,
'HAVING': TokenType.HAVING,
'COUNT': TokenType.COUNT,
'SUM': TokenType.SUM,
'AVG': TokenType.AVG,
'MIN': TokenType.MIN,
'MAX': TokenType.MAX,
'DISTINCT': TokenType.DISTINCT,
// v0.3.0
'BEGIN': TokenType.BEGIN,
'COMMIT': TokenType.COMMIT,
'ROLLBACK': TokenType.ROLLBACK,
'UNION': TokenType.UNION,
'ALL': TokenType.ALL,
'INDEX': TokenType.INDEX,
// v0.3.1
'CASE': TokenType.CASE,
'WHEN': TokenType.WHEN,
'THEN': TokenType.THEN,
'ELSE': TokenType.ELSE,
'END': TokenType.END,
// v0.5.1
'EXPLAIN': TokenType.EXPLAIN,
'ANALYZE': TokenType.ANALYZE,
'REINDEX': TokenType.REINDEX,
'VACUUM': TokenType.VACUUM,
'SAVEPOINT': TokenType.SAVEPOINT,
'RELEASE': TokenType.RELEASE,
'TO': TokenType.TO,
};
/**
* metona-sqlark SQL Lexer — 词法分析器
* @module sql/lexer
*
* 将 SQL 字符串切分为 Token 流。
*/
// ---------------------------------------------------------------------------
// Lexer
// ---------------------------------------------------------------------------
class Lexer {
constructor(input) {
this.position = 0;
this.readPosition = 0;
this.ch = '';
this.input = input;
this.readChar();
}
/** 读取下一个 Token */
nextToken() {
this.skipWhitespace();
let tok;
switch (this.ch) {
case ',':
tok = this.makeToken(TokenType.COMMA, ',');
break;
case '(':
tok = this.makeToken(TokenType.LPAREN, '(');
break;
case ')':
tok = this.makeToken(TokenType.RPAREN, ')');
break;
case ';':
tok = this.makeToken(TokenType.SEMICOLON, ';');
break;
case '*':
tok = this.makeToken(TokenType.STAR, '*');
break;
case '.':
tok = this.makeToken(TokenType.DOT, '.');
break;
case '=':
tok = this.makeToken(TokenType.EQ, '=');
break;
case '!':
if (this.peekChar() === '=') {
this.readChar();
tok = this.makeToken(TokenType.NEQ, '!=');
}
else {
tok = this.makeToken(TokenType.ILLEGAL, '!');
}
break;
case '>':
if (this.peekChar() === '=') {
this.readChar();
tok = this.makeToken(TokenType.GTE, '>=');
}
else {
tok = this.makeToken(TokenType.GT, '>');
}
break;
case '<':
if (this.peekChar() === '=') {
this.readChar();
tok = this.makeToken(TokenType.LTE, '<=');
}
else if (this.peekChar() === '>') {
this.readChar();
tok = this.makeToken(TokenType.NEQ, '<>');
}
else {
tok = this.makeToken(TokenType.LT, '<');
}
break;
case "'":
case '"':
tok = this.readString(this.ch);
break;
case '':
tok = { type: TokenType.EOF, value: '', position: this.position };
break;
default:
// SQL 注释: -- 行注释
if (this.ch === '-' && this.peekChar() === '-') {
this.skipLineComment();
return this.nextToken();
}
// SQL 注释: /* 块注释 */
if (this.ch === '/' && this.peekChar() === '*') {
this.skipBlockComment();
return this.nextToken();
}
if (this.isLetter(this.ch)) {
const ident = this.readIdentifier();
const keyword = KEYWORDS[ident.toUpperCase()];
tok = {
type: keyword ?? TokenType.IDENTIFIER,
value: ident,
position: this.position - ident.length,
};
return tok; // 已读取完毕,不需要再 readChar
}
else if (this.isDigit(this.ch) || (this.ch === '-' && this.isDigit(this.peekChar()))) {
const num = this.readNumber();
tok = {
type: TokenType.NUMBER,
value: num,
position: this.position - num.length,
};
return tok;
}
else {
tok = this.makeToken(TokenType.ILLEGAL, this.ch);
}
break;
}
this.readChar();
return tok;
}
// ---- 内部 ----
readChar() {
if (this.readPosition >= this.input.length) {
this.ch = '';
}
else {
this.ch = this.input[this.readPosition];
}
this.position = this.readPosition;
this.readPosition++;
}
peekChar() {
if (this.readPosition >= this.input.length)
return '';
return this.input[this.readPosition];
}
skipWhitespace() {
while (this.ch === ' ' || this.ch === '\t' || this.ch === '\n' || this.ch === '\r') {
this.readChar();
}
}
/** 跳过 -- 行注释到行尾 */
skipLineComment() {
while (this.ch !== '\n' && this.ch !== '\r' && this.ch !== '') {
this.readChar();
}
}
/** 跳过块注释 slash-star ... star-slash */
skipBlockComment() {
this.readChar(); // skip *
this.readChar(); // move past *
while (this.ch !== '' && !(this.ch === '*' && this.peekChar() === '/')) {
this.readChar();
}
if (this.ch !== '') {
this.readChar(); // skip *
this.readChar(); // skip /
}
}
readIdentifier() {
const start = this.position;
while (this.isLetter(this.ch) || this.isDigit(this.ch) || this.ch === '_') {
this.readChar();
}
return this.input.slice(start, this.position);
}
readNumber() {
const start = this.position;
// 负号
if (this.ch === '-')
this.readChar();
while (this.isDigit(this.ch)) {
this.readChar();
}
// 小数点
if (this.ch === '.' && this.isDigit(this.peekChar())) {
this.readChar();
while (this.isDigit(this.ch)) {
this.readChar();
}
}
return this.input.slice(start, this.position);
}
readString(quote) {
const start = this.position + 1; // 跳过一个引号
this.readChar(); // 跳过开始引号
let value = '';
while (this.ch !== '') {
if (this.ch === quote) {
// v0.3.3: 支持 SQL 标准 '' 转义(两个连续引号 = 一个引号)
if (this.peekChar() === quote) {
value += quote;
this.readChar(); // 跳过第二个引号
this.readChar();
continue;
}
break; // 结束引号(由 nextToken 的 readChar 跳过)
}
// 反斜杠转义(兼容旧语法)
if (this.ch === '\\' && this.peekChar() === quote) {
this.readChar();
value += quote;
this.readChar();
continue;
}
value += this.ch;
this.readChar();
}
// v0.7.2: 未闭合字符串字面量显式报错(此前静默返回残缺 STRING token
// 上层可解析出错误结果,如 `SELECT 'abc` 被当作合法常量列)
if (this.ch === '') {
throw new DatabaseError(`Unterminated string literal at position ${start}`, 'PARSE_ERROR');
}
return {
type: TokenType.STRING,
value,
position: start,
};
}
isLetter(ch) {
return /[a-zA-Z_]/.test(ch);
}
isDigit(ch) {
return /[0-9]/.test(ch);
}
makeToken(type, value) {
return { type, value, position: this.position };
}
}
// ---------------------------------------------------------------------------
// 便捷方法:一次性词法分析
// ---------------------------------------------------------------------------
/** 将 SQL 字符串解析为 Token 列表 */
function tokenize(sql) {
const lexer = new Lexer(sql);
const tokens = [];
let tok = lexer.nextToken();
while (tok.type !== TokenType.EOF) {
tokens.push(tok);
tok = lexer.nextToken();
}
tokens.push(tok); // EOF
return tokens;
}
/**
* metona-sqlark SQL Parser — 递归下降语法分析器
* @module sql/parser
*
* Token 流 → AST Statement。
* 支持的语法是标准 SQL 的子集。
*/
// ---------------------------------------------------------------------------
// Parser
// ---------------------------------------------------------------------------
class Parser {
constructor(sql) {
this.sql = sql;
this.lexer = new Lexer(sql);
// 预读两个 token
this.nextToken();
this.nextToken();
}
/** 解析完整 SQL 语句 */
parseStatement() {
switch (this.curToken.type) {
case TokenType.SELECT:
return this.parseSelect();
case TokenType.INSERT:
return this.parseInsert();
case TokenType.UPDATE:
return this.parseUpdate();
case TokenType.DELETE:
return this.parseDelete();
case TokenType.CREATE:
return this.parseCreateStatement();
case TokenType.DROP:
return this.parseDropStatement();
case TokenType.ALTER:
return this.parseAlterTable();
case TokenType.TRUNCATE:
return this.parseTruncateTable();
case TokenType.BEGIN:
return this.parseBegin();
case TokenType.COMMIT:
return this.parseCommit();
case TokenType.ROLLBACK:
return this.parseRollback();
// v0.5.1: 维护语句入口
case TokenType.EXPLAIN:
return this.parseExplain();
case TokenType.ANALYZE:
return this.parseAnalyze();
case TokenType.REINDEX:
return this.parseReindex();
case TokenType.VACUUM:
return this.parseVacuum();
case TokenType.SAVEPOINT:
return this.parseSavepoint();
case TokenType.RELEASE:
return this.parseSavepoint();
default:
throw this.error(`Unexpected token "${this.curToken.value}"`);
}
}
/** 解析所有语句(分号分隔的多语句支持) */
parseAllStatements() {
const statements = [];
while (!this.curTokenIs(TokenType.EOF)) {
// 跳过多余的分号
while (this.curTokenIs(TokenType.SEMICOLON))
this.nextToken();
if (this.curTokenIs(TokenType.EOF))
break;
statements.push(this.parseStatement());
// 语句后应紧跟分号或 EOF
if (this.curTokenIs(TokenType.SEMICOLON)) {
this.nextToken();
}
else if (!this.curTokenIs(TokenType.EOF)) {
throw this.error(`Expected ';' after statement, got "${this.curToken.value}"`);
}
}
return statements;
}
// ---- 维护语句(v0.5.1 ----
/** EXPLAIN <statement> — 输出查询计划 */
parseExplain() {
this.expect(TokenType.EXPLAIN);
if (this.curTokenIs(TokenType.EXPLAIN)) {
throw this.error('Nested EXPLAIN is not allowed');
}
const query = this.parseStatement();
return { type: 'EXPLAIN', query };
}
/** ANALYZE [TABLE] name — 收集表统计信息 */
parseAnalyze() {
this.expect(TokenType.ANALYZE);
if (this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'TABLE') {
this.nextToken();
}
return { type: 'ANALYZE', table: this.expectIdentifier('table name') };
}
/** REINDEX [TABLE] name — 重建表二级索引 */
parseReindex() {
this.expect(TokenType.REINDEX);
if (this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'TABLE') {
this.nextToken();
}
return { type: 'REINDEX', table: this.expectIdentifier('table name') };
}
/** VACUUM — 压缩 LSM + 清理碎片 */
parseVacuum() {
this.expect(TokenType.VACUUM);
return { type: 'VACUUM' };
}
/** SAVEPOINT name | RELEASE [SAVEPOINT] name */
parseSavepoint() {
let action;
if (this.curTokenIs(TokenType.RELEASE)) {
action = 'RELEASE';
this.nextToken();
}
else {
action = 'SAVE';
this.expect(TokenType.SAVEPOINT);
}
// 可选 SAVEPOINT 关键字(RELEASE SAVEPOINT name
if (this.curTokenIs(TokenType.SAVEPOINT))
this.nextToken();
return { type: 'SAVEPOINT', name: this.expectIdentifier('savepoint name'), action };
}
// ---- 事务语句 ----
parseBegin() {
this.expect(TokenType.BEGIN);
// 可选 TRANSACTION 关键字
if (this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'TRANSACTION') {
this.nextToken();
}
return { type: 'BEGIN' };
}
parseCommit() {
this.expect(TokenType.COMMIT);
if (this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'TRANSACTION') {
this.nextToken();
}
return { type: 'COMMIT' };
}
/** ROLLBACK [TRANSACTION] | ROLLBACK TO [SAVEPOINT] namev0.5.1 */
parseRollback() {
this.expect(TokenType.ROLLBACK);
if (this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'TRANSACTION') {
this.nextToken();
return { type: 'ROLLBACK' };
}
// ROLLBACK TO [SAVEPOINT] name
if (this.curTokenIs(TokenType.TO) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'TO')) {
this.nextToken();
if (this.curTokenIs(TokenType.SAVEPOINT) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'SAVEPOINT')) {
this.nextToken();
}
return { type: 'SAVEPOINT', name: this.expectIdentifier('savepoint name'), action: 'ROLLBACK' };
}
return { type: 'ROLLBACK' };
}
// ---- CREATE TABLE / CREATE INDEX ----
parseCreateStatement() {
this.expect(TokenType.CREATE);
if (this.curTokenIs(TokenType.TABLE)) {
return this.parseCreateTable();
}
if (this.curTokenIs(TokenType.INDEX) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'INDEX')) {
return this.parseCreateIndex();
}
if (this.curTokenIs(TokenType.UNIQUE)) {
// CREATE UNIQUE INDEX
this.nextToken();
if (this.curTokenIs(TokenType.INDEX) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'INDEX')) {
const stmt = this.parseCreateIndex();
stmt.unique = true;
return stmt;
}
}
throw this.error(`Expected TABLE or INDEX after CREATE, got "${this.curToken.value}"`);
}
parseCreateIndex() {
this.expect(TokenType.INDEX);
const name = this.expectIdentifier('index name');
this.expect(TokenType.ON);
const table = this.expectIdentifier('table name');
this.expect(TokenType.LPAREN);
const column = this.expectIdentifier('column name');
this.expect(TokenType.RPAREN);
return { type: 'CREATE_INDEX', name, table, column };
}
// ---- DROP TABLE / DROP INDEX ----
parseDropStatement() {
this.expect(TokenType.DROP);
if (this.curTokenIs(TokenType.TABLE)) {
return this.parseDropTable();
}
if (this.curTokenIs(TokenType.INDEX) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'INDEX')) {
return this.parseDropIndex();
}
throw this.error(`Expected TABLE or INDEX after DROP, got "${this.curToken.value}"`);
}
parseDropIndex() {
this.expect(TokenType.INDEX);
const name = this.expectIdentifier('index name');
// SQLite 风格:DROP INDEX idx_name [ON table]
let table = '';
let column = '';
if (this.curTokenIs(TokenType.ON)) {
this.nextToken();
table = this.expectIdentifier('table name');
if (this.curTokenIs(TokenType.LPAREN)) {
this.nextToken();
column = this.expectIdentifier('column name');
this.expect(TokenType.RPAREN);
}
}
return { type: 'DROP_INDEX', name, table, column };
}
// ===================================================================
// SELECT
// ===================================================================
parseSelect() {
this.expect(TokenType.SELECT);
// DISTINCT(可选)
let distinct = false;
if (this.curTokenIs(TokenType.DISTINCT)) {
distinct = true;
this.nextToken();
}
// 列
const columns = [];
if (this.curTokenIs(TokenType.STAR)) {
columns.push('*');
this.nextToken();
}
else {
columns.push(...this.parseColumnList());
}
// FROMv0.4.0 可选:SELECT 1 / SELECT 'lit' 无表查询)
let fromSubquery;
let tableName = '';
let alias;
if (this.curTokenIs(TokenType.FROM)) {
this.nextToken();
// v0.4.0: FROM (SELECT ...) AS alias 派生表
if (this.curTokenIs(TokenType.LPAREN)) {
this.nextToken();
fromSubquery = this.parseSelect();
this.expect(TokenType.RPAREN);
if (this.curTokenIs(TokenType.AS)) {
this.nextToken();
alias = this.expectIdentifier('alias');
}
else if (this.curToken.type === TokenType.IDENTIFIER && !this._isReservedAfterFrom()) {
alias = this.curToken.value;
this.nextToken();
}
}
else {
tableName = this.expectIdentifier('table name');
// 表别名(可选)
if (this.curTokenIs(TokenType.AS)) {
this.nextToken();
alias = this.expectIdentifier('alias');
}
else if (this.curToken.type === TokenType.IDENTIFIER && !this._isReservedAfterFrom()) {
alias = this.curToken.value;
this.nextToken();
}
}
}
const stmt = {
type: 'SELECT',
columns,
distinct: distinct || undefined,
from: tableName,
alias,
where: {},
};
if (fromSubquery) {
stmt.fromSubquery = fromSubquery;
}
// JOIN 子句(可选,支持多个)
const joins = this.parseJoinClauses();
if (joins.length > 0) {
stmt.joins = joins;
}
// WHERE(可选)
if (this.curTokenIs(TokenType.WHERE)) {
this.nextToken();
stmt.where = this.parseCondition();
}
// GROUP BY(可选)
if (this.curTokenIs(TokenType.GROUP)) {
this.nextToken();
this.expect(TokenType.BY);
stmt.groupBy = this.parseIdentifierList();
}
// HAVING(可选)
if (this.curTokenIs(TokenType.HAVING)) {
this.nextToken();
stmt.having = this.parseCondition();
}
// ORDER BY(可选)
if (this.curTokenIs(TokenType.ORDER)) {
this.nextToken();
this.expect(TokenType.BY);
stmt.orderBy = this.parseOrderByList();
}
// LIMIT(可选)
if (this.curTokenIs(TokenType.LIMIT)) {
this.nextToken();
stmt.limit = this.expectNumber('LIMIT value');
}
// OFFSET(可选)
if (this.curTokenIs(TokenType.OFFSET)) {
this.nextToken();
stmt.offset = this.expectNumber('OFFSET value');
}
// UNION / UNION ALL(可选,v0.3.0
if (this.curTokenIs(TokenType.UNION)) {
return this.parseUnion(stmt);
}
return stmt;
}
/** 解析 UNION / UNION ALL 组合(支持链式) */
parseUnion(left) {
this.expect(TokenType.UNION);
let all = false;
if (this.curTokenIs(TokenType.ALL)) {
all = true;
this.nextToken();
}
const right = this.parseSelect();
const unionStmt = { type: 'SELECT_UNION', left, right, all: all || undefined };
// 链式 UNION
if (this.curTokenIs(TokenType.UNION)) {
return this.parseUnionChain(unionStmt);
}
return unionStmt;
}
/** 链式 UNION:左侧是已组合的 UNION 语句 */
parseUnionChain(left) {
this.expect(TokenType.UNION);
let all = false;
if (this.curTokenIs(TokenType.ALL)) {
all = true;
this.nextToken();
}
const right = this.parseSelect();
const unionStmt = { type: 'SELECT_UNION', left, right, all: all || undefined };
if (this.curTokenIs(TokenType.UNION)) {
return this.parseUnionChain(unionStmt);
}
return unionStmt;
}
/** 解析 JOIN 子句列表 */
parseJoinClauses() {
const joins = [];
while (this._isJoinKeyword()) {
joins.push(this.parseJoinClause());
}
return joins;
}
_isJoinKeyword() {
return (this.curTokenIs(TokenType.INNER) ||
this.curTokenIs(TokenType.LEFT) ||
this.curTokenIs(TokenType.RIGHT) ||
this.curTokenIs(TokenType.CROSS) ||
this.curTokenIs(TokenType.JOIN));
}
/** 解析单个 JOIN 子句 */
parseJoinClause() {
let type = 'INNER';
if (this.curTokenIs(TokenType.INNER)) {
type = 'INNER';
this.nextToken();
}
else if (this.curTokenIs(TokenType.LEFT)) {
type = 'LEFT';
this.nextToken();
if (this.curTokenIs(TokenType.OUTER))
this.nextToken(); // 可选 OUTER
}
else if (this.curTokenIs(TokenType.RIGHT)) {
type = 'RIGHT';
this.nextToken();
if (this.curTokenIs(TokenType.OUTER))
this.nextToken();
}
else if (this.curTokenIs(TokenType.CROSS)) {
type = 'CROSS';
this.nextToken();
}
this.expect(TokenType.JOIN);
const tableName = this.expectIdentifier('table name');
// JOIN 表别名(可选)
let alias;
if (this.curTokenIs(TokenType.AS)) {
this.nextToken();
alias = this.expectIdentifier('alias');
}
else if (this.curToken.type === TokenType.IDENTIFIER && !this._isJoinReserved()) {
alias = this.curToken.value;
this.nextToken();
}
// ON 条件(CROSS JOIN 不需要 ON
let on = {};
if (type !== 'CROSS' && this.curTokenIs(TokenType.ON)) {
this.nextToken();
on = this.parseCondition();
}
return { type, table: tableName, alias, on };
}
/** 判断当前 token 是否为 FROM 之后的保留字 */
_isReservedAfterFrom() {
return (this.curTokenIs(TokenType.WHERE) ||
this.curTokenIs(TokenType.ORDER) ||
this.curTokenIs(TokenType.LIMIT) ||
this.curTokenIs(TokenType.OFFSET) ||
this.curTokenIs(TokenType.GROUP) ||
this._isJoinKeyword());
}
_isJoinReserved() {
return (this.curTokenIs(TokenType.ON) ||
this.curTokenIs(TokenType.WHERE) ||
this.curTokenIs(TokenType.ORDER) ||
this.curTokenIs(TokenType.LIMIT) ||
this._isJoinKeyword());
}
// ===================================================================
// INSERT
// ===================================================================
parseInsert() {
this.expect(TokenType.INSERT);
this.expect(TokenType.INTO);
const tableName = this.expectIdentifier('table name');
// 列名(可选)
let columns;
if (this.curTokenIs(TokenType.LPAREN)) {
this.nextToken();
columns = this.parseIdentifierList();
this.expect(TokenType.RPAREN);
}
// INSERT INTO ... SELECT ...v0.3.0
if (this.curTokenIs(TokenType.SELECT)) {
return {
type: 'INSERT',
into: tableName,
columns,
select: this.parseSelect(),
};
}
// VALUES
this.expect(TokenType.VALUES);
// 值列表
const values = [];
do {
if (this.curTokenIs(TokenType.COMMA)) {
this.nextToken();
}
this.expect(TokenType.LPAREN);
const rowValues = this.parseValueList();
this.expect(TokenType.RPAREN);
values.push(rowValues);
} while (this.curTokenIs(TokenType.COMMA));
return {
type: 'INSERT',
into: tableName,
columns,
values,
};
}
// ===================================================================
// UPDATE
// ===================================================================
parseUpdate() {
this.expect(TokenType.UPDATE);
const tableName = this.expectIdentifier('table name');
this.expect(TokenType.SET);
// SET col=val, ...
const sets = {};
do {
if (this.curTokenIs(TokenType.COMMA))
this.nextToken();
const col = this.expectIdentifier('column name');
this.expect(TokenType.EQ);
sets[col] = this.parseValue();
} while (this.curTokenIs(TokenType.COMMA));
let where = {};
if (this.curTokenIs(TokenType.WHERE)) {
this.nextToken();
where = this.parseCondition();
}
return { type: 'UPDATE', table: tableName, sets, where };
}
// ===================================================================
// DELETE
// ===================================================================
parseDelete() {
this.expect(TokenType.DELETE);
this.expect(TokenType.FROM);
const tableName = this.expectIdentifier('table name');
let where = {};
if (this.curTokenIs(TokenType.WHERE)) {
this.nextToken();
where = this.parseCondition();
}
return { type: 'DELETE', from: tableName, where };
}
// ===================================================================
// CREATE TABLE
// ===================================================================
parseCreateTable() {
this.expect(TokenType.TABLE);
// IF NOT EXISTS(可选)
let ifNotExists = false;
if (this.curTokenIs(TokenType.IF)) {
this.nextToken();
this.expect(TokenType.NOT);
this.expect(TokenType.EXISTS);
ifNotExists = true;
}
const tableName = this.expectIdentifier('table name');
this.expect(TokenType.LPAREN);
const columns = [];
do {
if (this.curTokenIs(TokenType.COMMA))
this.nextToken();
columns.push(this.parseColumnDef());
} while (this.curTokenIs(TokenType.COMMA));
this.expect(TokenType.RPAREN);
return { type: 'CREATE_TABLE', name: tableName, columns, ifNotExists: ifNotExists || undefined };
}
parseColumnDef() {
const name = this.expectIdentifier('column name');
const type = this.expectIdentifier('column type').toLowerCase();
const col = { name, type };
// 修饰符
while (this.curTokenIs(TokenType.PRIMARY) ||
this.curTokenIs(TokenType.UNIQUE) ||
this.curTokenIs(TokenType.NOT) ||
this.curTokenIs(TokenType.DEFAULT) ||
this.curTokenIs(TokenType.REFERENCES)) {
if (this.curTokenIs(TokenType.PRIMARY)) {
this.nextToken();
this.expect(TokenType.KEY);
col.primaryKey = true;
}
else if (this.curTokenIs(TokenType.UNIQUE)) {
this.nextToken();
col.unique = true;
}
else if (this.curTokenIs(TokenType.NOT)) {
this.nextToken();
this.expect(TokenType.NULL);
col.required = true;
}
else if (this.curTokenIs(TokenType.DEFAULT)) {
this.nextToken();
col.default = this.parseValue();
}
else if (this.curTokenIs(TokenType.REFERENCES)) {
this.nextToken();
const refTable = this.expectIdentifier('referenced table');
this.expect(TokenType.LPAREN);
const refCol = this.expectIdentifier('referenced column');
this.expect(TokenType.RPAREN);
col.references = `${refTable}.${refCol}`;
// ON DELETE / ON UPDATE
while (this.curTokenIs(TokenType.ON)) {
this.nextToken();
if (this.curTokenIs(TokenType.DELETE)) {
this.nextToken();
col.onDelete = this.parseCascadeAction();
}
else if (this.curTokenIs(TokenType.UPDATE)) {
this.nextToken();
col.onUpdate = this.parseCascadeAction();
}
else {
break;
}
}
}
else {
break;
}
}
return col;
}
/** 解析 CASCADE | SET NULL | RESTRICT */
parseCascadeAction() {
if (this.curTokenIs(TokenType.CASCADE)) {
this.nextToken();
return 'CASCADE';
}
if (this.curTokenIs(TokenType.SET)) {
this.nextToken();
this.expect(TokenType.NULL);
return 'SET NULL';
}
// RESTRICT 或默认
if (this.curToken.type === TokenType.IDENTIFIER && this.curToken.value.toUpperCase() === 'RESTRICT') {
this.nextToken();
return 'RESTRICT';
}
return 'RESTRICT';
}
// ===================================================================
// ALTER TABLE
// ===================================================================
parseAlterTable() {
this.expect(TokenType.ALTER);
this.expect(TokenType.TABLE);
const tableName = this.expectIdentifier('table name');
// ADD COLUMN / DROP COLUMN
let action;
if (this.curTokenIs(TokenType.ADD)) {
action = 'ADD';
this.nextToken();
// Optional COLUMN keyword
if (this.curToken.type === TokenType.IDENTIFIER && this.curToken.value.toUpperCase() === 'COLUMN') {
this.nextToken();
}
const col = this.parseColumnDef();
return { type: 'ALTER_TABLE', name: tableName, action, column: col };
}
else if (this.curTokenIs(TokenType.DROP) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'DROP')) {
action = 'DROP';
this.nextToken();
// Optional COLUMN keyword
if (this.curToken.type === TokenType.IDENTIFIER && this.curToken.value.toUpperCase() === 'COLUMN') {
this.nextToken();
}
const colName = this.expectIdentifier('column name');
return { type: 'ALTER_TABLE', name: tableName, action, column: { name: colName, type: 'string' } };
}
else {
throw this.error('Expected ADD or DROP in ALTER TABLE');
}
}
// ===================================================================
// TRUNCATE TABLE
// ===================================================================
parseTruncateTable() {
this.expect(TokenType.TRUNCATE);
this.expect(TokenType.TABLE);
const tableName = this.expectIdentifier('table name');
return { type: 'TRUNCATE_TABLE', name: tableName };
}
// ===================================================================
// DROP TABLE
// ===================================================================
parseDropTable() {
this.expect(TokenType.TABLE);
// IF EXISTS(可选)
let ifExists = false;
if (this.curTokenIs(TokenType.IF)) {
this.nextToken();
this.expect(TokenType.EXISTS);
ifExists = true;
}
const tableName = this.expectIdentifier('table name');
return { type: 'DROP_TABLE', name: tableName, ifExists: ifExists || undefined };
}
// ===================================================================
// 条件表达式
// ===================================================================
/** condition → simple_cond ((AND|OR) simple_cond)* */
parseCondition() {
let left = this.parseSimpleCondition();
while (this.curTokenIs(TokenType.AND) || this.curTokenIs(TokenType.OR)) {
const isAnd = this.curTokenIs(TokenType.AND);
this.nextToken();
const right = this.parseSimpleCondition();
if (isAnd) {
// 合并到 $and
left = { $and: [left, right] };
}
else {
left = { $or: [left, right] };
}
}
return left;
}
/** 公共 WHERE 条件入口(供 CASE WHEN 求值等外部场景,v0.3.1 */
parseWhere() {
return this.parseCondition();
}
/** simple_cond → column op value | column IS [NOT] NULL | column [NOT] LIKE pattern
* | column [NOT] IN (values) | NOT condition | (condition)
* | [NOT] EXISTS (SELECT ...) ← v0.3.0 */
parseSimpleCondition() {
// [NOT] EXISTS (SELECT ...)
if (this.curTokenIs(TokenType.EXISTS) ||
(this._isKeywordAsIdent() && this.curToken.value.toUpperCase() === 'EXISTS')) {
this.nextToken();
return this.parseExistsCondition(false);
}
if (this.curTokenIs(TokenType.NOT) && this._peekIsExists()) {
this.nextToken(); // 跳过 NOT
this.nextToken(); // 跳过 EXISTS
return this.parseExistsCondition(true);
}
// NOT expr(注意 NOT IN / NOT LIKE 不作为通用 NOT
if (this.curTokenIs(TokenType.NOT) && !this._isNotInOrLike()) {
this.nextToken();
const inner = this.parseSimpleCondition();
return { $not: inner };
}
// (condition)
if (this.curTokenIs(TokenType.LPAREN)) {
this.nextToken();
const inner = this.parseCondition();
this.expect(TokenType.RPAREN);
return inner;
}
// column
const column = this.parseColumnRef();
// IS NULL / IS NOT NULL
if (this.curTokenIs(TokenType.IDENTIFIER) && this.curToken.value.toUpperCase() === 'IS') {
this.nextToken();
const isNot = this.curTokenIs(TokenType.NOT);
if (isNot)
this.nextToken();
this.expect(TokenType.NULL);
const result = {};
result[column] = isNot ? { $ne: null } : { $eq: null };
return result;
}
// BETWEEN val1 AND val2
if (this.curTokenIs(TokenType.BETWEEN)) {
this.nextToken();
const low = this.parseValue();
this.expect(TokenType.AND);
const high = this.parseValue();
const result = {};
result[column] = { $gte: low, $lte: high };
return result;
}
// NOT BETWEEN val1 AND val2
if (this.curTokenIs(TokenType.NOT) && this.peekTokenIs(TokenType.BETWEEN)) {
this.nextToken(); // skip NOT
this.nextToken(); // skip BETWEEN
const low = this.parseValue();
this.expect(TokenType.AND);
const high = this.parseValue();
const result = {};
result[column] = { $not: { $gte: low, $lte: high } };
return result;
}
// NOT LIKE / NOT INNOT 后紧跟 LIKE 或 IN
if (this.curTokenIs(TokenType.NOT)) {
if (this.peekTokenIs(TokenType.IN)) {
// NOT IN
this.nextToken(); // skip NOT
this.nextToken(); // skip IN
this.expect(TokenType.LPAREN);
if (this.curTokenIs(TokenType.SELECT)) {
const subquery = this.parseSelect();
this.expect(TokenType.RPAREN);
const result = {};
result[column] = { $nin: { $subquery: subquery } };
return result;
}
const values = this.parseValueList();
this.expect(TokenType.RPAREN);
const result = {};
result[column] = { $nin: values };
return result;
}
else if (this.peekTokenIs(TokenType.LIKE)) {
// NOT LIKE
this.nextToken(); // skip NOT
this.nextToken(); // skip LIKE
const pattern = this.parseValue();
const result = {};
result[column] = { $not: { $like: pattern } };
return result;
}
}
// LIKE
if (this.curTokenIs(TokenType.LIKE)) {
this.nextToken();
const pattern = this.parseValue();
const result = {};
result[column] = { $like: pattern };
return result;
}
// IN
if (this.curTokenIs(TokenType.IN)) {
this.nextToken();
this.expect(TokenType.LPAREN);
// 子查询: IN (SELECT ...)
if (this.curTokenIs(TokenType.SELECT)) {
const subquery = this.parseSelect();
this.expect(TokenType.RPAREN);
const result = {};
result[column] = { $in: { $subquery: subquery } };
return result;
}
const values = this.parseValueList();
this.expect(TokenType.RPAREN);
const result = {};
result[column] = { $in: values };
return result;
}
// v0.4.1: 裸布尔列条件(WHERE done / CASE WHEN done THEN)— 列后直接是终止符时视为真值判断
if (this.curTokenIs(TokenType.AND) || this.curTokenIs(TokenType.OR) ||
this.curTokenIs(TokenType.RPAREN) || this.curTokenIs(TokenType.EOF) ||
(this.curToken.type === TokenType.IDENTIFIER && ['THEN', 'END', 'ELSE', 'NULLS', 'LIMIT', 'OFFSET', 'ORDER', 'GROUP', 'HAVING', 'UNION', 'WHERE'].includes(this.curToken.value.toUpperCase()))) {
const result = {};
result[column] = { $eq: true };
return result;
}
// 比较运算符
const op = this.parseComparisonOp();
// 子查询: op (SELECT ...)
if (this.curTokenIs(TokenType.LPAREN) && this.peekTokenIs(TokenType.SELECT)) {
this.nextToken(); // skip (
const subquery = this.parseSelect();
this.expect(TokenType.RPAREN);
const result = {};
result[column] = { [op]: { $subquery: subquery } };
return result;
}
// 尝试解析列引用(identifier DOT identifier 格式)
let value;
if ((this.curToken.type === TokenType.IDENTIFIER || this._isKeywordAsIdent()) &&
this.peekTokenIs(TokenType.DOT)) {
const colRef = this.parseColumnRef();
value = { $col: colRef };
}
else {
value = this.parseValue();
}
const result = {};
result[column] = { [op]: value };
return result;
}
/** 解析 EXISTS (SELECT ...) / NOT EXISTS (SELECT ...) */
parseExistsCondition(negate) {
this.expect(TokenType.LPAREN);
const subquery = this.parseSelect();
this.expect(TokenType.RPAREN);
// $exists 键由 Executor.resolveSubqueries 解析为 booleanwhere-matcher 消费
return { $exists: { $subquery: subquery, $negate: negate || undefined } };
}
/** 判断当前 NOT 后是否紧跟 EXISTS */
_peekIsExists() {
return this.peekToken.type === TokenType.EXISTS ||
(this.peekToken.type === TokenType.IDENTIFIER && this.peekToken.value.toUpperCase() === 'EXISTS');
}
/** 判断当前 NOT 是否为 NOT IN / NOT LIKE 的一部分(不应作为通用 NOT 处理) */
_isNotInOrLike() {
return this.peekTokenIs(TokenType.IN) || this.peekTokenIs(TokenType.LIKE);
}
peekTokenIs(type) {
return this.peekToken.type === type;
}
parseComparisonOp() {
switch (this.curToken.type) {
case TokenType.EQ:
this.nextToken();
return '$eq';
case TokenType.NEQ:
this.nextToken();
return '$ne';
case TokenType.GT:
this.nextToken();
return '$gt';
case TokenType.GTE:
this.nextToken();
return '$gte';
case TokenType.LT:
this.nextToken();
return '$lt';
case TokenType.LTE:
this.nextToken();
return '$lte';
default:
throw this.error(`Expected comparison operator, got "${this.curToken.value}"`);
}
}
// ===================================================================
// 辅助解析
// ===================================================================
parseColumnList() {
const cols = [];
cols.push(this.parseColumnWithAlias());
while (this.curTokenIs(TokenType.COMMA)) {
this.nextToken();
cols.push(this.parseColumnWithAlias());
}
return cols;
}
/** v0.3.3: 解析列(支持 `col AS alias` 显式别名与 `col alias` 隐式别名) */
parseColumnWithAlias() {
let col = this.parseColumnRef();
if (this.curTokenIs(TokenType.AS)) {
this.nextToken();
const alias = this.expectIdentifier('alias');
col = `${col} AS ${alias}`;
}
else if (this.curToken.type === TokenType.IDENTIFIER && !this._isReservedAfterFrom() && !this._isJoinKeyword()) {
const alias = this.curToken.value;
this.nextToken();
col = `${col} AS ${alias}`;
}
return col;
}
/** 解析列引用:支持 'col'、'table.col'、'COUNT(*)'/'SUM(col)'、数字常量列(SELECT 1)、字符串常量列(SELECT 'x'v0.4.0)和 CASE WHEN 表达式(v0.3.1 */
parseColumnRef() {
// CASE WHEN 表达式(v0.3.1
if (this.curTokenIs(TokenType.CASE)) {
return this.parseCaseExpressionText();
}
// 数字常量列:SELECT 1 FROM t(常见于 EXISTS 子查询)
if (this.curTokenIs(TokenType.NUMBER)) {
const value = this.curToken.value;
this.nextToken();
return value;
}
// v0.4.0: 字符串常量列:SELECT 'value' FROM t
if (this.curTokenIs(TokenType.STRING)) {
const value = this.curToken.value;
this.nextToken();
return `'${value}'`;
}
// 聚合函数?
if (this.curTokenIs(TokenType.COUNT) ||
this.curTokenIs(TokenType.SUM) ||
this.curTokenIs(TokenType.AVG) ||
this.curTokenIs(TokenType.MIN) ||
this.curTokenIs(TokenType.MAX)) {
return this.parseAggregateCall();
}
const first = this.expectIdentifier('column name');
if (this.curTokenIs(TokenType.DOT)) {
this.nextToken();
const second = this.expectIdentifier('column name');
return `${first}.${second}`;
}
return first;
}
/**
* 解析 CASE WHEN 表达式,返回原文(含可选 AS 别名)。
* 例:CASE WHEN age > 30 THEN 'senior' ELSE 'junior' END AS status
*/
parseCaseExpressionText() {
const start = this.curToken.position;
this.nextToken(); // 跳过 CASE
let depth = 1;
let end = start + 'CASE'.length;
while (!this.curTokenIs(TokenType.EOF) && depth > 0) {
if (this.curTokenIs(TokenType.CASE))
depth++;
if (this.curTokenIs(TokenType.END)) {
depth--;
end = this.curToken.position + 'END'.length;
this.nextToken();
if (depth === 0)
break;
}
end = this.curToken.position + this.curToken.value.length;
this.nextToken();
}
let text = this.sql.slice(start, end);
// 可选 AS 别名
if (this.curTokenIs(TokenType.AS)) {
this.nextToken();
text += ` AS ${this.expectIdentifier('alias')}`;
}
else if (this.curToken.type === TokenType.IDENTIFIER && !this.curTokenIs(TokenType.COMMA) && !this._isReservedAfterFrom()) {
text += ` AS ${this.curToken.value}`;
this.nextToken();
}
return text;
}
/** 解析聚合函数调用: COUNT(*), SUM(col), AVG(col), MIN(col), MAX(col)v0.4.0 支持 COUNT(DISTINCT col) */
parseAggregateCall() {
const func = this.curToken.value.toUpperCase();
this.nextToken();
this.expect(TokenType.LPAREN);
// v0.4.0: COUNT(DISTINCT col) 等去重聚合
let distinct = false;
if (this.curTokenIs(TokenType.DISTINCT)) {
distinct = true;
this.nextToken();
}
let arg;
if (this.curTokenIs(TokenType.STAR)) {
arg = '*';
this.nextToken();
}
else {
arg = this.parseColumnRef();
}
this.expect(TokenType.RPAREN);
// 可选别名: AS alias
let alias = '';
if (this.curTokenIs(TokenType.AS)) {
this.nextToken();
alias = this.expectIdentifier('alias');
}
else if (this.curToken.type === TokenType.IDENTIFIER && this._isAggregateAlias()) {
alias = this.curToken.value;
this.nextToken();
}
const inner = distinct ? `DISTINCT ${arg}` : arg;
if (alias) {
return `${func}(${inner}) AS ${alias}`;
}
return `${func}(${inner})`;
}
_isAggregateAlias() {
return !this._isReservedAfterFrom() && !this._isJoinKeyword();
}
parseIdentifierList() {
const ids = [];
ids.push(this.parseIdentifierWithDot());
while (this.curTokenIs(TokenType.COMMA)) {
this.nextToken();
ids.push(this.parseIdentifierWithDot());
}
return ids;
}
parseValueList() {
const vals = [];
vals.push(this.parseValue());
while (this.curTokenIs(TokenType.COMMA)) {
this.nextToken();
vals.push(this.parseValue());
}
return vals;
}
parseOrderByList() {
const list = [];
list.push(this.parseOrderBy());
while (this.curTokenIs(TokenType.COMMA)) {
this.nextToken();
list.push(this.parseOrderBy());
}
return list;
}
parseOrderBy() {
const column = this.parseIdentifierWithDot();
let direction = 'asc';
if (this.curTokenIs(TokenType.ASC)) {
this.nextToken();
}
else if (this.curTokenIs(TokenType.DESC)) {
direction = 'desc';
this.nextToken();
}
// v0.4.0: NULLS FIRST / NULLS LAST
let nulls;
if (this.curTokenIs(TokenType.IDENTIFIER) && this.curToken.value.toUpperCase() === 'NULLS') {
this.nextToken();
if (this.curTokenIs(TokenType.IDENTIFIER) && this.curToken.value.toUpperCase() === 'FIRST') {
nulls = 'first';
this.nextToken();
}
else if (this.curTokenIs(TokenType.IDENTIFIER) && this.curToken.value.toUpperCase() === 'LAST') {
nulls = 'last';
this.nextToken();
}
}
return { column, direction, ...(nulls ? { nulls } : {}) };
}
/** v0.4.0: 标识符(支持 'table.column' 带表前缀引用,用于 ORDER BY / GROUP BY */
parseIdentifierWithDot() {
const first = this.expectIdentifier('identifier');
if (this.curTokenIs(TokenType.DOT)) {
this.nextToken();
return `${first}.${this.expectIdentifier('identifier')}`;
}
return first;
}
/** 解析字面量值 */
parseValue() {
switch (this.curToken.type) {
case TokenType.STRING: {
const val = this.curToken.value;
this.nextToken();
return val;
}
case TokenType.NUMBER: {
const val = Number(this.curToken.value);
this.nextToken();
return val;
}
case TokenType.TRUE:
this.nextToken();
return true;
case TokenType.FALSE:
this.nextToken();
return false;
case TokenType.NULL:
this.nextToken();
return null;
default:
throw this.error(`Expected value, got "${this.curToken.value}"`);
}
}
// ===================================================================
// Token 操作
// ===================================================================
nextToken() {
this.curToken = this.peekToken;
this.peekToken = this.lexer.nextToken();
}
curTokenIs(type) {
return this.curToken.type === type;
}
expect(type) {
if (this.curTokenIs(type)) {
this.nextToken();
return;
}
throw this.error(`Expected ${type}, got "${this.curToken.value}"`);
}
expectIdentifier(context) {
if (this.curToken.type === TokenType.IDENTIFIER || this._isKeywordAsIdent()) {
const val = this.curToken.value;
this.nextToken();
return val;
}
throw this.error(`Expected ${context}, got "${this.curToken.value}"`);
}
/** 关键字可以作为标识符(如列名等于关键字) */
_isKeywordAsIdent() {
return (this.curToken.type !== TokenType.EOF &&
this.curToken.type !== TokenType.ILLEGAL &&
this.curToken.type !== TokenType.STRING &&
this.curToken.type !== TokenType.NUMBER &&
this.curToken.type !== TokenType.COMMA &&
this.curToken.type !== TokenType.LPAREN &&
this.curToken.type !== TokenType.RPAREN &&
this.curToken.type !== TokenType.SEMICOLON &&
this.curToken.type !== TokenType.EQ &&
this.curToken.type !== TokenType.NEQ &&
this.curToken.type !== TokenType.GT &&
this.curToken.type !== TokenType.GTE &&
this.curToken.type !== TokenType.LT &&
this.curToken.type !== TokenType.LTE &&
this.curToken.type !== TokenType.DOT &&
this.curToken.type !== TokenType.STAR);
}
expectNumber(context) {
if (this.curToken.type === TokenType.NUMBER) {
const val = Number(this.curToken.value);
this.nextToken();
return val;
}
throw this.error(`Expected ${context}, got "${this.curToken.value}"`);
}
error(msg) {
return new DatabaseError(`Parse error at position ${this.curToken.position}: ${msg}`, 'PARSE_ERROR');
}
}
// ---------------------------------------------------------------------------
// 便捷方法
// ---------------------------------------------------------------------------
/** 解析 SQL 字符串为 AST Statement */
function parse(sql) {
const parser = new Parser(sql);
const stmt = parser.parseStatement();
return stmt;
}
/** 解析 SQL 字符串为 AST Statement 数组(分号分隔的多语句支持,v0.3.0) */
function parseAll(sql) {
const parser = new Parser(sql);
return parser.parseAllStatements();
}
/** 解析独立 WHERE 条件表达式(CASE WHEN 求值等场景,v0.3.1 */
function parseWhereCondition(sql) {
const parser = new Parser(sql);
return parser.parseWhere();
}
/**
* metona-sqlark Query Executor — AST 执行器
* @module query/executor
*
* JOIN / GROUP BY / DISTINCT 逻辑在此层处理。
*/
/** 解析 "CASE WHEN c1 THEN v1 WHEN c2 THEN v2 ELSE v3 END [AS alias]" */
function parseCaseExpression(expr) {
const m = expr.match(/^\s*CASE\s+([\s\S]*?)\s+END\s*(?:AS\s+(\w+))?\s*$/i);
if (!m)
return null;
const body = m[1];
const alias = m[2] ?? null;
const whens = [];
const re = /WHEN\s+([\s\S]*?)\s+THEN\s+([\s\S]*?)(?=\s+WHEN\s+|\s+ELSE\s+|\s*$)/gi;
let match;
while ((match = re.exec(body)) !== null) {
let cond = null;
try {
cond = parseWhereCondition(match[1].trim());
}
catch {
// 条件解析失败视为不匹配
}
whens.push({ cond, value: match[2].trim() });
}
let elseValue = null;
const elseMatch = body.match(/\sELSE\s+([\s\S]*)$/i);
if (elseMatch)
elseValue = elseMatch[1].trim();
return { whens, elseValue, alias };
}
/** 解析 CASE 值:字面量(null/true/false/数字/字符串)优先,其次列引用 → 行值 */
function resolveCaseValue(text, row) {
const v = text.trim();
if (v === 'null')
return null;
if (v === 'true')
return true;
if (v === 'false')
return false;
const num = Number(v);
if (v !== '' && !isNaN(num))
return num;
const str = v.match(/^'(.*)'$/s) || v.match(/^"(.*)"$/s);
if (str)
return str[1];
if (/^[a-zA-Z_][a-zA-Z0-9_.]*$/.test(v)) {
return row[v] ?? null; // 列引用(含 table.col
}
return v;
}
/** 对行求值 CASE WHEN 表达式 */
function evaluateCase(expr, row) {
for (const { cond, value } of expr.whens) {
if (cond && matchWhere(row, cond)) {
return resolveCaseValue(value, row);
}
}
return expr.elseValue !== null ? resolveCaseValue(expr.elseValue, row) : null;
}
// ---------------------------------------------------------------------------
// Executor
// ---------------------------------------------------------------------------
class QueryExecutor {
constructor(engine, maxRowsPerQuery = 0) {
this.engine = engine;
this.maxRowsPerQuery = maxRowsPerQuery;
}
async execute(stmt) {
switch (stmt.type) {
case 'SELECT': return this.executeSelect(stmt);
case 'SELECT_UNION': return this.executeSelectUnion(stmt);
case 'EXPLAIN': return this.executeExplain(stmt);
case 'INSERT': return this.executeInsert(stmt);
case 'UPDATE': return this.executeUpdate(stmt);
case 'DELETE': return this.executeDelete(stmt);
case 'CREATE_TABLE': return this.executeCreateTable(stmt);
case 'DROP_TABLE': return this.executeDropTable(stmt);
case 'ALTER_TABLE': return this.executeAlterTable(stmt);
case 'TRUNCATE_TABLE': return this.executeTruncateTable(stmt);
case 'CREATE_INDEX': return this.executeCreateIndex(stmt);
case 'DROP_INDEX': return this.executeDropIndex(stmt);
case 'BEGIN': return this.executeBegin();
case 'COMMIT': return this.executeCommit();
case 'ROLLBACK': return this.executeRollback();
// v0.5.1: 维护语句
case 'SAVEPOINT': return this.executeSavepoint(stmt);
case 'ANALYZE': return this.executeAnalyze(stmt);
case 'REINDEX': return this.executeReindex(stmt);
case 'VACUUM': return this.executeVacuum();
default: throw new DatabaseError('Unknown statement type', 'UNKNOWN_STATEMENT');
}
}
// ===================================================================
// UNIONv0.3.0
// ===================================================================
/** 递归执行 UNION / UNION ALL,返回合并结果 */
async executeSelectUnion(stmt) {
const leftRows = await this.executeSelectPart(stmt.left);
const rightRows = await this.executeSelectPart(stmt.right);
// 列名以左侧为准,右侧只取值
const leftCols = leftRows.length > 0 ? Object.keys(leftRows[0]) : [];
const normalized = leftRows.map((row) => row);
if (stmt.all) {
for (const row of rightRows)
normalized.push(this.projectUnionRow(row, leftCols));
return normalized;
}
// UNION 去重(与 DISTINCT 相同的列值拼接键)
const seen = new Set();
const result = [];
for (const row of normalized) {
const key = Object.values(row).map((v) => String(v ?? '\0')).join('\x1f');
if (!seen.has(key)) {
seen.add(key);
result.push(row);
}
}
for (const row of rightRows) {
const projected = this.projectUnionRow(row, leftCols);
const key = Object.values(projected).map((v) => String(v ?? '\0')).join('\x1f');
if (!seen.has(key)) {
seen.add(key);
result.push(projected);
}
}
return result;
}
async executeSelectPart(part) {
return part.type === 'SELECT_UNION' ? this.executeSelectUnion(part) : this.executeSelect(part);
}
/** 将 UNION 右侧行投影为左侧列结构(按位置取值) */
projectUnionRow(row, leftCols) {
if (leftCols.length === 0)
return row;
const values = Object.values(row);
const projected = {};
for (let i = 0; i < leftCols.length; i++) {
projected[leftCols[i]] = i < values.length ? values[i] : null;
}
return projected;
}
/** EXPLAIN: 输出查询计划 */
async executeExplain(stmt) {
const startTime = Date.now();
let result = null;
let rows = 0;
// v0.6.2-fix: EXPLAIN 不得真实执行写语句 —— 此前 EXPLAIN DELETE/UPDATE 会产生
// 真实副作用(删/改数据)。仅 SELECT 类语句执行(只读);UPDATE/DELETE 用
// count 估算影响行数(无副作用);INSERT/DDL 仅输出计划不执行。
if (stmt.query.type === 'SELECT' || stmt.query.type === 'SELECT_UNION') {
try {
result = await this.execute(stmt.query);
}
catch { /* explain 即使执行失败也返回计划 */ }
rows = Array.isArray(result) ? result.length : 0;
}
else if (stmt.query.type === 'UPDATE' || stmt.query.type === 'DELETE') {
try {
const plan = compileStatement(stmt.query);
rows = await this.engine.count(plan.table, plan);
}
catch {
rows = 0;
}
}
const elapsed = Date.now() - startTime;
// v0.5.1: 仅 SELECT/DELETE/UPDATE 有引擎查询计划;其他语句输出基本信息
let plan = null;
try {
plan = compileStatement(stmt.query);
}
catch { /* 非查询语句无 QueryPlan */ }
// v0.7.0: 真实索引命中信息(此前 usingIndex 恒为 'auto' 占位)。
// 引擎无关启发式:WHERE 中存在主键/索引/唯一列条件 → 对应引擎索引路径。
let usingIndex = plan?.table ? 'none' : 'none';
if (plan && plan.table && plan.where && Object.keys(plan.where).length > 0) {
try {
const schema = await this.engine.getTableSchema(plan.table);
if (schema) {
for (const col of Object.keys(plan.where)) {
if (col.startsWith('$'))
continue;
const colDef = schema.columns[col];
if (!colDef)
continue;
if (colDef.primaryKey) {
usingIndex = 'pk';
break;
}
if (colDef.index || colDef.unique) {
usingIndex = `index:${col}`;
break;
}
}
}
}
catch { /* schema 读取失败保持 none */ }
}
return {
type: stmt.query.type,
table: plan?.table,
columns: plan?.columns,
where: plan?.where || {},
orderBy: plan?.orderBy || [],
limit: plan?.limit,
offset: plan?.offset,
usingIndex,
estimatedRows: rows,
actualTimeMs: elapsed,
};
}
// ===================================================================
// SELECT
// ===================================================================
async executeSelect(stmt) {
const hasGroupBy = !!(stmt.groupBy && stmt.groupBy.length > 0);
const hasAggregate = !hasGroupBy && this._hasAggregateColumn(stmt.columns);
let rows;
const isJoinQuery = !!(stmt.joins && stmt.joins.length > 0);
// CASE WHEN 表达式需要原始列求值(SELECT 列或 WHERE 条件中的 CASE):
// 引擎层取全行,投影统一在 executor 端完成
const needsRawRows = this.hasCaseColumn(stmt.columns) ||
(!!stmt.where && this.whereHasCase(stmt.where));
// v0.3.3: ORDER BY 引用 SELECT 别名 → 引擎层不排序/不截断,投影后再排序
const orderByAlias = this.orderByUsesSelectAlias(stmt);
// v0.3.3: SELECT 列含 `col AS alias` → 引擎层投影会丢失源列,统一取原始行由 executor 投影
const hasSelectAlias = stmt.columns.some((c) => /\s+AS\s+\w+$/i.test(c));
if (stmt.fromSubquery) {
// v0.4.0: FROM (SELECT ...) 派生表 — 子查询结果作为行源
const subRows = await this.executeSelectPart(stmt.fromSubquery);
rows = isJoinQuery
? await this.executeJoinSelect(stmt, subRows.map((row) => this.prefixRow(row, stmt.alias ?? '')))
: subRows;
if (!isJoinQuery && stmt.where && Object.keys(stmt.where).length > 0) {
// 非 JOINWHERE 在 executor 端过滤(子查询结果不经引擎)
stmt.where = await this.resolveSubqueries(stmt.where);
rows = rows.filter((row) => matchWhere(row, stmt.where));
}
}
else if (!stmt.from && !isJoinQuery) {
// v0.4.0: 无表查询(SELECT 1 / SELECT 'lit')— 单行空上下文,常量列投影
rows = [{}];
}
else if (isJoinQuery) {
// JOIN 路径:行带表别名前缀(如 'd.id'),WHERE 保持原名不剥离
rows = await this.executeJoinSelect(stmt);
}
else {
// 非 JOIN 路径:规范化 WHERE 字段名(剥离主表别名前缀,修复 WHERE u.age > 20
if (stmt.where && Object.keys(stmt.where).length > 0) {
stmt.where = this.normalizeWhereColumns(stmt.where, [stmt.alias ?? stmt.from]);
}
// v0.4.0: ORDER BY / GROUP BY 带表前缀同样剥离(如 ORDER BY u.age
const mainAliases = [stmt.alias ?? stmt.from].filter(Boolean);
if (stmt.orderBy && stmt.orderBy.length > 0) {
stmt.orderBy = stmt.orderBy.map((o) => ({ ...o, column: this.stripAlias(o.column, mainAliases) }));
}
if (stmt.groupBy && stmt.groupBy.length > 0) {
stmt.groupBy = stmt.groupBy.map((c) => this.stripAlias(c, mainAliases));
}
// v0.4.0: SELECT 列带表前缀剥离(SELECT u.name → name,行键无前缀)
stmt.columns = stmt.columns.map((c) => {
if (c === '*' || /^(COUNT|SUM|AVG|MIN|MAX)\(/i.test(c) || /^\s*CASE\b/i.test(c) || /^'/.test(c))
return c;
const m = c.match(/^(.+?)\s+AS\s+(\w+)$/i);
if (m) {
const stripped = this.stripAlias(m[1].trim(), mainAliases);
return stripped === m[1].trim() ? c : `${stripped} AS ${m[2]}`;
}
return this.stripAlias(c, mainAliases);
});
// WHERE 含关联子查询($col 引用外层行)→ 逐行绑定上下文求值
if (stmt.where && this.hasCorrelatedRefs(stmt.where)) {
const plan = compileStatement(hasGroupBy || hasAggregate ? { ...stmt, columns: ['*'] } : stmt);
// v0.4.0 修复: 关联子查询需要完整外层行(SELECT 列可能不含被 $col 引用的列,如 EXISTS 绑定的主键)
plan.columns = ['*'];
if (orderByAlias) {
plan.orderBy = undefined;
plan.limit = undefined;
plan.offset = undefined;
}
rows = await this.engine.find(plan.table, { ...plan, where: this.stripCorrelatedExists(stmt.where) });
rows = await this.filterCorrelated(rows, stmt.where);
}
else {
// 先解析子查询
if (stmt.where && Object.keys(stmt.where).length > 0) {
stmt.where = await this.resolveSubqueries(stmt.where);
}
const plan = compileStatement(hasGroupBy || hasAggregate ? { ...stmt, columns: ['*'] } : stmt);
if (needsRawRows || hasSelectAlias)
plan.columns = ['*'];
if (orderByAlias) {
plan.orderBy = undefined;
plan.limit = undefined;
plan.offset = undefined;
}
rows = await this.engine.find(plan.table, plan);
}
}
// 无 GROUP BY 但有聚合 → 计算单行聚合结果
if (hasAggregate) {
rows = [this.computeSingleAggregate(rows, stmt)];
}
if (hasGroupBy)
rows = this.executeGroupBy(rows, stmt);
if (stmt.distinct)
rows = this.executeDistinct(rows);
if (stmt.having && Object.keys(stmt.having).length > 0) {
// v0.4.0 修复: HAVING 中的标量子查询(HAVING SUM(o.amount) > (SELECT AVG(...)))需先解析
stmt.having = await this.resolveSubqueries(stmt.having);
// v0.4.0: HAVING 引用聚合表达式键(如 SUM(o.amount))时归一为别名键(如 spent)
const aliasMap = stmt._aggAliasMap;
if (aliasMap && aliasMap.size > 0) {
const normalized = {};
for (const [k, v] of Object.entries(stmt.having)) {
normalized[aliasMap.get(k) ?? k] = v;
}
stmt.having = normalized;
}
rows = rows.filter((row) => matchWhere(row, stmt.having));
}
if (stmt.orderBy && stmt.orderBy.length > 0)
rows = applyOrderBy(rows, stmt.orderBy);
if (!hasGroupBy && !hasAggregate && stmt.columns.length > 0 && stmt.columns[0] !== '*') {
rows = rows.map((row) => this.projectRow(row, stmt.columns));
}
// v0.3.3: ORDER BY 别名 → 投影后才存在,需在投影后重新排序
if (orderByAlias && stmt.orderBy && stmt.orderBy.length > 0) {
rows = applyOrderBy(rows, stmt.orderBy);
}
const offset = stmt.offset ?? 0;
const limit = stmt.limit ?? rows.length;
rows = rows.slice(offset, offset + limit);
// 全局行数上限保护
if (this.maxRowsPerQuery > 0 && rows.length > this.maxRowsPerQuery) {
rows = rows.slice(0, this.maxRowsPerQuery);
}
return rows;
}
// ---- JOIN ----
async executeJoinSelect(stmt, preloadedMain) {
const mainAlias = stmt.alias ?? stmt.from;
// v0.4.0: 派生表行源已预加载(行带别名前缀)
let mainRows;
if (preloadedMain) {
mainRows = preloadedMain;
}
else {
// v0.4.1: WHERE 中主表前缀等值条件下推到引擎(走二级索引,如 WHERE o.user_id = '1'
const { pushable } = this.extractPushableWhere(stmt.where ?? {}, mainAlias);
mainRows = (await this.engine.find(stmt.from, {
table: stmt.from,
where: Object.keys(pushable).length > 0 ? pushable : undefined,
})).map((row) => this.prefixRow(row, mainAlias));
}
let resultRows = mainRows;
for (const join of stmt.joins) {
const joinAlias = join.alias ?? join.table;
// v0.3.2: 等值 ON + 右列主键 → 哈希连接(一次 $in 查询替代嵌套循环)
const hashJoined = await this.tryHashJoin(resultRows, join, joinAlias, mainAlias);
if (hashJoined) {
resultRows = hashJoined;
continue;
}
const joinRows = (await this.engine.find(join.table, { table: join.table }))
.map((row) => this.prefixRow(row, joinAlias));
resultRows = this.joinRows(resultRows, joinRows, join);
}
if (stmt.where && Object.keys(stmt.where).length > 0) {
// v0.3.1: 关联子查询($col/EXISTS 引用外层行)→ 逐行绑定求值
if (this.hasCorrelatedRefs(stmt.where)) {
resultRows = await this.filterCorrelated(resultRows, stmt.where);
}
else {
// 非关联子查询(IN (SELECT ...) 等),字段名保持别名前缀
stmt.where = await this.resolveSubqueries(stmt.where);
resultRows = resultRows.filter((row) => matchWhere(row, stmt.where));
}
}
return resultRows;
}
prefixRow(row, alias) {
const prefixed = {};
for (const [key, value] of Object.entries(row))
prefixed[`${alias}.${key}`] = value;
return prefixed;
}
/**
* v0.4.1: 提取可下推的 WHERE 条件 — 主表别名前缀的普通条件(如 o.user_id = '1')。
* 下推到引擎可走二级索引;$col/$subquery/$and/$or/$not 等复杂条件保守不下推。
*/
extractPushableWhere(where, mainAlias) {
const pushable = {};
if (!mainAlias)
return { pushable };
const prefix = `${mainAlias}.`;
for (const [key, value] of Object.entries(where)) {
if (!key.startsWith(prefix))
continue;
const v = value;
if (typeof v === 'object' && v !== null &&
('$col' in v || '$subquery' in v || '$and' in v || '$or' in v || '$not' in v)) {
continue;
}
pushable[key.slice(prefix.length)] = value;
}
return { pushable };
}
/**
* 哈希连接(v0.3.2 单等值 / v0.4.0 多列等值):
* ON 为等值条件(单列或多列)且右表任一列为索引/主键时,
* 收集左表连接值 → 一次 $in 查询右表 → 哈希映射匹配。
* 替代嵌套循环,大表 INNER/LEFT JOIN 复杂度 O(N + M)。
* 不适用时返回 null(回退嵌套循环)。
*/
async tryHashJoin(leftRows, join, joinAlias, mainAlias) {
if (join.type === 'CROSS' || join.type === 'RIGHT')
return null;
// 解析 ON 为 (leftCol, rightCol) 等值对列表(v0.4.0 支持多列,含顶层 $and 展开)
const pairs = [];
const collectPairs = (on) => {
for (const [keyCol, cond] of Object.entries(on)) {
if (keyCol === '$and') {
if (!cond.every(collectPairs))
return false;
continue;
}
if (keyCol === '$or' || keyCol === '$not')
return false; // 非等值逻辑不适用
let refCol = null;
if (typeof cond === 'object' && cond !== null) {
const c = cond;
if ('$eq' in c && typeof c.$eq === 'object' && c.$eq !== null && '$col' in c.$eq) {
refCol = String(c.$eq.$col);
}
else if ('$col' in c && Object.keys(c).length === 1) {
refCol = String(c.$col);
}
}
if (!refCol)
return false; // 非等值条件不适用哈希连接
const keyIsLeft = mainAlias ? keyCol.startsWith(`${mainAlias}.`) : false;
pairs.push({
leftCol: keyIsLeft ? keyCol : refCol,
rightCol: keyIsLeft ? refCol : keyCol,
});
}
return true;
};
if (!collectPairs(join.on))
return null;
if (pairs.length === 0)
return null;
// 右表列必须是主键/索引列(确保 $in 走索引)——任一列即可
const schema = await this.engine.getTableSchema(join.table);
if (!schema)
return null;
const probePair = pairs.find((p) => {
const bare = p.rightCol.split('.').pop();
const colDef = schema.columns[bare];
return colDef && (colDef.primaryKey || colDef.index || colDef.unique);
});
if (!probePair)
return null;
// 收集左表连接值(去重)——用探测列的值缩小候选集
const probeRightBare = probePair.rightCol.split('.').pop();
const values = Array.from(new Set(leftRows.map((r) => r[probePair.leftCol]).filter((v) => v !== undefined && v !== null)));
if (values.length === 0)
return null;
// 一次 $in 查询右表(缩小候选集)
const rightRows = await this.engine.find(join.table, {
table: join.table,
where: { [probeRightBare]: { $in: values } },
});
// 构建复合键哈希映射:右表多列值 → 行列表
const hash = new Map();
for (const rr of rightRows) {
const key = pairs.map((p) => String(rr[p.rightCol.split('.').pop()] ?? '\0')).join('\x1f');
if (!hash.has(key))
hash.set(key, []);
hash.get(key).push(rr);
}
const nullRight = {};
for (const key of Object.keys(schema.columns))
nullRight[key] = null;
const result = [];
for (const l of leftRows) {
const key = pairs.map((p) => String(l[p.leftCol] ?? '\0')).join('\x1f');
const matches = hash.get(key);
if (matches && matches.length > 0) {
for (const r of matches) {
result.push({ ...l, ...this.prefixRow(r, joinAlias) });
}
}
else if (join.type === 'LEFT') {
// LEFT JOIN 无匹配 → 右表列置 null
result.push({ ...l, ...this.prefixRow(nullRight, joinAlias) });
}
// INNER JOIN 无匹配 → 跳过
}
return result;
}
/** 嵌套循环连接(优化:避免 ON 时对象扩散) */
joinRows(leftRows, rightRows, join) {
if (join.type === 'CROSS') {
const result = [];
for (const l of leftRows)
for (const r of rightRows)
result.push({ ...l, ...r });
return result;
}
const result = [];
for (const l of leftRows) {
let matched = false;
for (const r of rightRows) {
// 合并后匹配 ON(避免创建临时对象再丢弃)
const merged = { ...l, ...r };
if (matchWhere(merged, join.on, { $col: true })) {
result.push(merged);
matched = true;
}
}
if (!matched && join.type === 'LEFT') {
const nullRight = {};
for (const key of Object.keys(rightRows[0] ?? {}))
nullRight[key] = null;
result.push({ ...l, ...nullRight });
}
}
if (join.type === 'RIGHT') {
for (const r of rightRows) {
const isMatched = leftRows.some((l) => {
const merged = { ...l, ...r };
return matchWhere(merged, join.on, { $col: true });
});
if (!isMatched) {
const nullLeft = {};
for (const key of Object.keys(leftRows[0] ?? {}))
nullLeft[key] = null;
result.push({ ...nullLeft, ...r });
}
}
}
return result;
}
// ---- GROUP BY ----
executeGroupBy(rows, stmt) {
const groups = new Map();
for (const row of rows) {
const key = stmt.groupBy.map((col) => String(row[col] ?? 'null')).join('|');
if (!groups.has(key))
groups.set(key, []);
groups.get(key).push(row);
}
const result = [];
// v0.4.0: 聚合表达式键 → 输出键 映射(HAVING SUM(...) 引用表达式时归一为别名键)
const aliasMap = new Map();
for (const groupRows of groups.values()) {
const aggregated = {};
for (const col of stmt.groupBy)
aggregated[col] = groupRows[0][col];
for (const colExpr of stmt.columns) {
if (colExpr === '*')
continue;
const m = colExpr.match(/^(COUNT|SUM|AVG|MIN|MAX)\((.+?)\)(?:\s+AS\s+(\w+))?$/i);
if (m) {
const [, func, arg, alias] = m;
const value = this.computeAggregate(func.toUpperCase(), groupRows, arg.trim());
const exprKey = `${func.toUpperCase()}(${arg.trim()})`;
const outputKey = alias || colExpr;
if (outputKey !== exprKey)
aliasMap.set(exprKey, outputKey);
aggregated[outputKey] = value;
}
else if (/^\s*CASE\b/i.test(colExpr)) {
// v0.3.2: 非聚合的 CASE WHEN 列取组内第一行求值
const expr = parseCaseExpression(colExpr);
aggregated[expr?.alias ?? colExpr] = expr ? evaluateCase(expr, groupRows[0]) : null;
}
else if (!stmt.groupBy.includes(colExpr)) {
aggregated[colExpr] = groupRows[0][colExpr];
}
}
result.push(aggregated);
}
stmt._aggAliasMap = aliasMap;
return result;
}
computeAggregate(func, rows, col) {
// v0.3.2: 聚合参数支持 CASE WHEN 表达式(如 SUM(CASE WHEN age > 18 THEN 1 ELSE 0 END)
const caseExpr = /^\s*CASE\b/i.test(col) ? parseCaseExpression(col) : null;
// v0.4.0: COUNT(DISTINCT col) 等去重聚合
const distinctArg = !caseExpr && /^\s*DISTINCT\s+/i.test(col);
const argCol = distinctArg ? col.replace(/^\s*DISTINCT\s+/i, '').trim() : col;
const rawValues = rows
.map((r) => (caseExpr ? evaluateCase(caseExpr, r) : r[argCol]))
.filter((v) => v !== null && v !== undefined);
// v0.4.0: COUNT 对原始值去重(任意类型);数值聚合在类型转换后去重
if (func === 'COUNT') {
if (argCol === '*')
return rows.length;
if (distinctArg) {
return new Set(rawValues.map((v) => (typeof v === 'object' ? JSON.stringify(v) : String(v)))).size;
}
return rawValues.length;
}
const nums = rawValues.map(Number);
const distinctNums = distinctArg ? Array.from(new Set(nums)) : nums;
switch (func) {
case 'SUM': return distinctNums.reduce((a, b) => a + b, 0);
case 'AVG': return distinctNums.length === 0 ? 0 : distinctNums.reduce((a, b) => a + b, 0) / distinctNums.length;
case 'MIN': return distinctNums.length === 0 ? 0 : Math.min(...distinctNums);
case 'MAX': return distinctNums.length === 0 ? 0 : Math.max(...distinctNums);
default: return 0;
}
}
// ---- DISTINCT(优化:列值拼接代替 JSON.stringify ----
executeDistinct(rows) {
const seen = new Set();
return rows.filter((row) => {
const key = Object.values(row).map((v) => String(v ?? '\0')).join('\x1f');
if (seen.has(key))
return false;
seen.add(key);
return true;
});
}
// ===================================================================
// 其他语句
// ===================================================================
async executeInsert(stmt) {
const schema = await this.engine.getTableSchema(stmt.into);
if (!schema)
throw new DatabaseError(`Table "${stmt.into}" does not exist`, 'TABLE_NOT_FOUND');
const colNames = stmt.columns ?? Object.keys(schema.columns);
// INSERT INTO ... SELECT ...v0.3.0
if (stmt.select) {
const selectRows = await this.executeSelectPart(stmt.select);
// v0.4.0 修复:源列顺序不能依赖行键(validateRow 会跳过 undefined 导致行键缺失/乱序)。
// 以 SELECT 列列表 / 源表 schema 列顺序为准,按位置对齐目标列,缺列不填。
let srcCols = [];
const sel = stmt.select;
if (sel.type === 'SELECT') {
if (sel.columns && sel.columns.length > 0 && sel.columns[0] !== '*') {
srcCols = sel.columns.map((c) => c.split('.').pop());
}
else if (sel.from) {
const srcSchema = await this.engine.getTableSchema(sel.from);
srcCols = srcSchema ? Object.keys(srcSchema.columns) : [];
}
}
if (srcCols.length === 0 && selectRows.length > 0) {
srcCols = Object.keys(selectRows[0]);
}
const rows = selectRows.map((row) => {
const mapped = {};
for (let i = 0; i < colNames.length; i++) {
const src = i < srcCols.length ? srcCols[i] : null;
if (src && src in row)
mapped[colNames[i]] = row[src];
}
return mapped;
});
return this.engine.insert(stmt.into, rows);
}
const rows = (stmt.values ?? []).map((vals) => {
const row = {};
for (let i = 0; i < colNames.length; i++) {
if (i < vals.length)
row[colNames[i]] = vals[i];
}
return row;
});
return this.engine.insert(stmt.into, rows);
}
async executeUpdate(stmt) {
const plan = compileStatement(stmt);
return this.engine.update(plan.table, plan, stmt.sets);
}
async executeDelete(stmt) {
const plan = compileStatement(stmt);
return this.engine.delete(plan.table, plan);
}
async executeCreateTable(stmt) {
// IF NOT EXISTS: 表已存在时静默返回
if (stmt.ifNotExists) {
const exists = await this.engine.hasTable(stmt.name);
if (exists)
return;
}
// v0.7.1: Object.create(null) —— 防止 '__proto__' 列名触发原型 setter 静默丢列
// createSchema 校验会显式拒绝该列名)
const columns = Object.create(null);
for (const col of stmt.columns)
columns[col.name] = astColumnToColumnDef(col);
return this.engine.createTable(createSchema(stmt.name, columns));
}
async executeDropTable(stmt) {
if (stmt.ifExists) {
const exists = await this.engine.hasTable(stmt.name);
if (!exists)
return; // IF EXISTS: 表不存在时静默返回
}
return this.engine.dropTable(stmt.name);
}
async executeAlterTable(stmt) {
const exists = await this.engine.hasTable(stmt.name);
if (!exists)
throw new DatabaseError(`Table "${stmt.name}" does not exist`, 'TABLE_NOT_FOUND');
const schema = await this.engine.getTableSchema(stmt.name);
if (!schema)
return;
// v0.7.0: ALTER ADD 主键列防护 —— 复合主键不支持(与 createSchema 校验对齐),
// 避免绕过建表校验添加第二个主键列导致语义陷阱
if (stmt.action === 'ADD' && stmt.column.primaryKey) {
const hasPk = Object.values(schema.columns).some((c) => c.primaryKey);
if (hasPk) {
throw new DatabaseError(`Composite primary keys are not supported yet: table "${stmt.name}" already has a primary key column`, 'SCHEMA_ERROR');
}
}
// v0.4.1: 引擎级 alterTableAria 需重写存储行 + 持久化 schema;其余引擎走通用引用路径)
if (typeof this.engine.alterTable === 'function') {
return this.engine.alterTable(stmt.name, stmt.action, { ...astColumnToColumnDef(stmt.column), name: stmt.column.name });
}
if (stmt.action === 'ADD') {
if (schema.columns[stmt.column.name]) {
throw new DatabaseError(`Column "${stmt.column.name}" already exists in table "${stmt.name}"`, 'COLUMN_EXISTS');
}
// 直接在 schema 引用上添加列(已有行的该列值为 undefined/default
schema.columns[stmt.column.name] = astColumnToColumnDef(stmt.column);
}
else if (stmt.action === 'DROP') {
if (!schema.columns[stmt.column.name]) {
throw new DatabaseError(`Column "${stmt.column.name}" does not exist in table "${stmt.name}"`, 'COLUMN_NOT_FOUND');
}
// 从 schema 引用上删除列定义(保留所有行数据)
delete schema.columns[stmt.column.name];
// 清除已有行中该列的值(MemoryEngine 的 find 返回引用,delete 直接生效)
const rows = await this.engine.find(stmt.name, { table: stmt.name });
const colName = stmt.column.name;
for (const row of rows) {
if (colName in row)
delete row[colName];
}
}
}
async executeTruncateTable(stmt) {
const exists = await this.engine.hasTable(stmt.name);
if (!exists)
throw new DatabaseError(`Table "${stmt.name}" does not exist`, 'TABLE_NOT_FOUND');
return this.engine.clear(stmt.name);
}
// ===================================================================
// CREATE INDEX / DROP INDEXv0.3.0
// ===================================================================
async executeCreateIndex(stmt) {
const exists = await this.engine.hasTable(stmt.table);
if (!exists)
throw new DatabaseError(`Table "${stmt.table}" does not exist`, 'TABLE_NOT_FOUND');
const schema = await this.engine.getTableSchema(stmt.table);
if (schema && !schema.columns[stmt.column]) {
throw new DatabaseError(`Column "${stmt.column}" does not exist in table "${stmt.table}"`, 'COLUMN_NOT_FOUND');
}
if (typeof this.engine.createIndex !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support CREATE INDEX`, 'NOT_SUPPORTED');
}
return this.engine.createIndex(stmt.table, stmt.column, stmt.unique);
}
async executeDropIndex(stmt) {
if (typeof this.engine.dropIndex !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support DROP INDEX`, 'NOT_SUPPORTED');
}
return this.engine.dropIndex(stmt.table, stmt.column, stmt.name);
}
// ===================================================================
// 事务语句(v0.3.0
// ===================================================================
async executeBegin() {
return this.engine.beginTransaction();
}
async executeCommit() {
return this.engine.commitTransaction();
}
async executeRollback() {
return this.engine.rollbackTransaction();
}
// ===================================================================
// 维护语句(v0.5.1
// ===================================================================
/** SAVEPOINT name / ROLLBACK TO SAVEPOINT name / RELEASE SAVEPOINT name */
async executeSavepoint(stmt) {
const engine = this.engine;
if (stmt.action === 'SAVE') {
if (typeof engine.savepoint !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support SAVEPOINT`, 'NOT_SUPPORTED');
}
return engine.savepoint(stmt.name);
}
if (stmt.action === 'ROLLBACK') {
if (typeof engine.rollbackToSavepoint !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support ROLLBACK TO SAVEPOINT`, 'NOT_SUPPORTED');
}
return engine.rollbackToSavepoint(stmt.name);
}
if (typeof engine.releaseSavepoint !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support RELEASE SAVEPOINT`, 'NOT_SUPPORTED');
}
return engine.releaseSavepoint(stmt.name);
}
/** ANALYZE TABLE name — 收集表统计信息 */
async executeAnalyze(stmt) {
const engine = this.engine;
if (typeof engine.analyzeTable !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support ANALYZE`, 'NOT_SUPPORTED');
}
const exists = await this.engine.hasTable(stmt.table);
if (!exists)
throw new DatabaseError(`Table "${stmt.table}" does not exist`, 'TABLE_NOT_FOUND');
return engine.analyzeTable(stmt.table);
}
/** REINDEX TABLE name — 重建表二级索引 */
async executeReindex(stmt) {
const engine = this.engine;
if (typeof engine.reindexTable !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support REINDEX`, 'NOT_SUPPORTED');
}
const exists = await this.engine.hasTable(stmt.table);
if (!exists)
throw new DatabaseError(`Table "${stmt.table}" does not exist`, 'TABLE_NOT_FOUND');
return engine.reindexTable(stmt.table);
}
/** VACUUM — 压缩 LSM + 清理碎片 */
async executeVacuum() {
const engine = this.engine;
if (typeof engine.vacuum !== 'function') {
throw new DatabaseError(`Engine "${this.engine.name}" does not support VACUUM`, 'NOT_SUPPORTED');
}
return engine.vacuum();
}
/** 列列表是否包含 CASE WHEN 表达式 */
hasCaseColumn(columns) {
return columns.some((col) => /^\s*CASE\b/i.test(col));
}
/**
* v0.3.3: ORDER BY 是否引用 SELECT 别名(如 `SELECT name AS n ... ORDER BY n`)。
* 别名列在引擎层投影前不存在,需投影后重新排序。
*/
orderByUsesSelectAlias(stmt) {
if (!stmt.orderBy || stmt.orderBy.length === 0)
return false;
const aliases = new Set();
for (const col of stmt.columns) {
const m = col.match(/\s+AS\s+(\w+)$/i);
if (m)
aliases.add(m[1]);
else if (/^\s*CASE\b/i.test(col)) {
const expr = parseCaseExpression(col);
if (expr?.alias)
aliases.add(expr.alias);
}
}
if (aliases.size === 0)
return false;
return stmt.orderBy.some((o) => aliases.has(o.column));
}
/** WHERE 是否包含 CASE WHEN 表达式键 */
whereHasCase(where) {
for (const [key, value] of Object.entries(where)) {
if (key === '$and' || key === '$or') {
if (value.some((sub) => this.whereHasCase(sub)))
return true;
continue;
}
if (key === '$not') {
if (this.whereHasCase(value))
return true;
continue;
}
if (/^\s*CASE\b/i.test(key))
return true;
}
return false;
}
getEngine() { return this.engine; }
/**
* 列投影(v0.3.1):普通列走 projectColumnsCASE WHEN 表达式逐行求值;
* v0.3.3: 支持 `col AS alias` 列别名
*/
projectRow(row, columns) {
const plain = [];
const aliasCols = [];
const caseCols = [];
const constCols = [];
for (const col of columns) {
if (col === '*')
continue;
const expr = parseCaseExpression(col);
if (expr) {
caseCols.push({ alias: expr.alias ?? col, expr });
continue;
}
const m = col.match(/^(.+?)\s+AS\s+(\w+)$/i);
if (m) {
aliasCols.push({ alias: m[2], source: m[1].trim() });
continue;
}
// v0.4.0: 字符串常量列 SELECT 'lit' → 常量输出
const lit = col.match(/^'(.*)'$/s);
if (lit) {
const value = lit[1].replace(/\\'/g, "'");
constCols.push({ key: col, value });
continue;
}
plain.push(col);
}
const projected = plain.length > 0 ? projectColumns(row, plain) : {};
for (const { alias, source } of aliasCols) {
if (source === '*') {
Object.assign(projected, row);
}
else {
const lit = source.match(/^'(.*)'$/s);
projected[alias] = lit ? lit[1].replace(/\\'/g, "'") : row[source];
}
}
for (const { key, value } of constCols) {
projected[key] = value;
}
for (const { alias, expr } of caseCols) {
projected[alias] = evaluateCase(expr, row);
}
return projected;
}
// ===================================================================
// 无 GROUP BY 时的聚合计算
// ===================================================================
/** 检查 SELECT 列列表中是否包含聚合函数 */
_hasAggregateColumn(columns) {
return columns.some((col) => /^(COUNT|SUM|AVG|MIN|MAX)\(/i.test(col));
}
/** 计算单行聚合结果(无 GROUP BY) */
computeSingleAggregate(rows, stmt) {
const result = {};
for (const colExpr of stmt.columns) {
if (colExpr === '*')
continue;
const m = colExpr.match(/^(COUNT|SUM|AVG|MIN|MAX)\((.+?)\)(?:\s+AS\s+(\w+))?$/i);
if (m) {
const [, func, arg, alias] = m;
result[alias || colExpr] = this.computeAggregate(func.toUpperCase(), rows, arg.trim());
}
else {
// 非聚合列取第一行的值
result[colExpr] = rows.length > 0 ? rows[0][colExpr] : null;
}
}
return result;
}
// ===================================================================
// 关联子查询 / 别名规范化(v0.3.0)
// ===================================================================
/** 剥离主表别名前缀:'u.id' → 'id'(键与 $col 值均处理,支持多层别名) */
normalizeWhereColumns(where, aliases) {
const normalized = {};
for (const [key, value] of Object.entries(where)) {
if (key === '$and' || key === '$or') {
normalized[key] = value.map((sub) => this.normalizeWhereColumns(sub, aliases));
continue;
}
if (key === '$not') {
normalized.$not = this.normalizeWhereColumns(value, aliases);
continue;
}
if (key === '$exists') {
normalized[key] = this.normalizeExistsValue(value, aliases);
continue;
}
const newKey = this.stripAlias(key, aliases);
normalized[newKey] = this.normalizeFieldValue(value, aliases);
}
return normalized;
}
normalizeExistsValue(value, aliases) {
if (typeof value !== 'object' || value === null)
return value;
const v = value;
if (v.$subquery) {
const sub = v.$subquery;
// 子查询 where 需同时识别:子查询自身别名 + 外层别名(关联引用)
const subAliases = [sub.alias ?? sub.from, ...aliases].filter(Boolean);
return { ...v, $subquery: { ...sub, where: this.normalizeWhereColumns(sub.where, subAliases) } };
}
return value;
}
normalizeFieldValue(value, aliases) {
if (typeof value !== 'object' || value === null || Array.isArray(value))
return value;
const ops = {};
for (const [op, operand] of Object.entries(value)) {
if (op === '$and' || op === '$or') {
ops[op] = operand.map((sub) => this.normalizeWhereColumns(sub, aliases));
}
else if (op === '$not' && typeof operand === 'object' && operand !== null) {
ops[op] = this.normalizeFieldValue(operand, aliases);
}
else if (op === '$col') {
ops[op] = this.stripAlias(String(operand), aliases);
}
else if (typeof operand === 'object' && operand !== null && !Array.isArray(operand) && '$col' in operand) {
// 操作符值中嵌套的列引用:{ $eq: { $col: 'u.id' } }
ops[op] = { $col: this.stripAlias(String(operand.$col), aliases) };
}
else {
ops[op] = operand;
}
}
return ops;
}
stripAlias(col, aliases) {
for (const alias of aliases) {
if (!alias)
continue;
const prefix = `${alias}.`;
if (col.startsWith(prefix))
return col.slice(prefix.length);
}
return col;
}
/** WHERE 是否含关联引用($col 或关联 EXISTS)或 CASE WHEN 表达式键 */
hasCorrelatedRefs(where) {
for (const [key, value] of Object.entries(where)) {
if (key === '$and' || key === '$or') {
if (value.some((sub) => this.hasCorrelatedRefs(sub)))
return true;
continue;
}
if (key === '$not') {
if (this.hasCorrelatedRefs(value))
return true;
continue;
}
if (key === '$exists') {
// 关联 EXISTS:子查询 where 含 $col 或主 where 含 $negate 未解析标记
if (typeof value === 'object' && value !== null && '$subquery' in value) {
return true; // 关联 EXISTS 统一走逐行求值
}
continue;
}
// v0.3.2: CASE WHEN 表达式键(逐行求值)
if (/^\s*CASE\b/i.test(key))
return true;
if (this.fieldHasColRef(value))
return true;
}
return false;
}
fieldHasColRef(value) {
if (typeof value !== 'object' || value === null || Array.isArray(value))
return false;
const ops = value;
if ('$col' in ops)
return true;
if ('$and' in ops || '$or' in ops) {
const subs = (ops.$and ?? ops.$or);
return subs.some((sub) => this.hasCorrelatedRefs(sub));
}
if ('$not' in ops && typeof ops.$not === 'object' && ops.$not !== null) {
return this.fieldHasColRef(ops.$not);
}
// 操作符值中嵌套的列引用:{ $eq: { $col: 'id' } }
for (const [, operand] of Object.entries(ops)) {
if (typeof operand === 'object' && operand !== null && !Array.isArray(operand)) {
if ('$col' in operand)
return true;
if (this.fieldHasColRef(operand))
return true;
}
}
return false;
}
/** 移除关联 EXISTS 标记(引擎层先执行无 EXISTS 条件的查询) */
stripCorrelatedExists(where) {
const cleaned = {};
for (const [key, value] of Object.entries(where)) {
if (key === '$and' || key === '$or') {
cleaned[key] = value.map((sub) => this.stripCorrelatedExists(sub));
continue;
}
if (key === '$not') {
const inner = this.stripCorrelatedExists(value);
// 剥离后为空 → 条件恒真,删掉该键(避免引擎层执行 NOT(true) 过滤掉所有行)
if (Object.keys(inner).length > 0)
cleaned.$not = inner;
continue;
}
if (key === '$exists')
continue; // 逐行求值时单独处理
if (/^\s*CASE\b/i.test(key))
continue; // v0.3.2: CASE 键逐行求值
cleaned[key] = value;
}
return cleaned;
}
/** 逐行绑定外层行上下文,求值关联 EXISTS、$col 引用与 CASE WHEN 键 */
async filterCorrelated(rows, where) {
const result = [];
for (const row of rows) {
// 1. CASE WHEN 键 → 布尔条件(同步)
let rowWhere = this.resolveCaseKeys(where, row);
// 2. $col 绑定 + 关联 EXISTS 求值(异步)
rowWhere = await this.resolveSubqueries(rowWhere, row);
if (matchWhere(row, rowWhere)) {
result.push(row);
}
}
return result;
}
/** 将 WHERE 中的 CASE WHEN 表达式键求值为布尔条件($caseResult */
resolveCaseKeys(where, row) {
const resolved = {};
for (const [key, value] of Object.entries(where)) {
if (key === '$and' || key === '$or') {
resolved[key] = value.map((sub) => this.resolveCaseKeys(sub, row));
continue;
}
if (key === '$not') {
resolved.$not = this.resolveCaseKeys(value, row);
continue;
}
if (/^\s*CASE\b/i.test(key)) {
const expr = parseCaseExpression(key);
if (!expr)
continue; // 解析失败视为不满足
const val = evaluateCase(expr, row);
if (this.caseConditionMatches(val, value)) {
resolved.$caseResult = true;
}
else {
return { $caseResult: false };
}
continue;
}
resolved[key] = value;
}
return resolved;
}
/** CASE 求值结果与操作符条件比较 */
caseConditionMatches(val, condition) {
if (typeof condition !== 'object' || condition === null || Array.isArray(condition)) {
return val === condition;
}
const ops = condition;
for (const [op, operand] of Object.entries(ops)) {
switch (op) {
case '$eq':
if (val !== operand)
return false;
break;
case '$ne':
if (val === operand)
return false;
break;
case '$gt':
if (!(val > operand))
return false;
break;
case '$gte':
if (!(val >= operand))
return false;
break;
case '$lt':
if (!(val < operand))
return false;
break;
case '$lte':
if (!(val <= operand))
return false;
break;
case '$in':
if (!(Array.isArray(operand) && operand.includes(val)))
return false;
break;
case '$nin':
if (Array.isArray(operand) && operand.includes(val))
return false;
break;
}
}
return true;
}
/** 将 where 中的 $col 引用替换为上下文行值 */
bindColumnRefs(value, contextRow) {
if (typeof value !== 'object' || value === null || Array.isArray(value))
return value;
const ops = {};
for (const [op, operand] of Object.entries(value)) {
if (op === '$col') {
ops[op] = contextRow[String(operand)] ?? null;
}
else if (op === '$and' || op === '$or') {
ops[op] = operand.map((sub) => this.bindWhereRefs(sub, contextRow));
}
else if (op === '$not' && typeof operand === 'object' && operand !== null) {
ops[op] = this.bindColumnRefs(operand, contextRow);
}
else if (typeof operand === 'object' && operand !== null && !Array.isArray(operand) && '$col' in operand) {
// 操作符值中嵌套的列引用:{ $eq: { $col: 'id' } } → { $eq: row['id'] }
ops[op] = contextRow[String(operand.$col)] ?? null;
}
else {
ops[op] = operand;
}
}
return ops;
}
bindWhereRefs(where, contextRow) {
const bound = {};
for (const [key, value] of Object.entries(where)) {
if (key === '$and' || key === '$or') {
bound[key] = value.map((sub) => this.bindWhereRefs(sub, contextRow));
}
else if (key === '$not') {
bound.$not = this.bindWhereRefs(value, contextRow);
}
else if (key === '$exists') {
bound[key] = value;
}
else {
bound[key] = this.bindColumnRefs(value, contextRow);
}
}
return bound;
}
// ===================================================================
// 子查询解析
// ===================================================================
/**
* 递归扫描 WHERE 条件,找到 $subquery 标记并执行子查询,
* 将结果替换为具体值。
* @param contextRow 关联子查询的外层行上下文(用于绑定 $col 引用)
*/
async resolveSubqueries(where, contextRow) {
// 关联上下文:先把字段级的 $col 引用绑定为外层行值
if (contextRow) {
where = this.bindWhereRefs(where, contextRow);
}
const resolved = {};
for (const [key, value] of Object.entries(where)) {
// 顶层 $existsv0.3.0):执行子查询并解析为 boolean,由 where-matcher 消费
if (key === '$exists' && typeof value === 'object' && value !== null) {
const v = value;
const sub = v.$subquery;
const negate = !!v.$negate;
let subWhere = sub.where;
// 子查询内的关联引用(如 o.user_id = u.id 中的 u.id)绑定外层行
if (this.hasCorrelatedRefs(subWhere)) {
subWhere = this.bindWhereRefs(subWhere, contextRow ?? {});
}
const rows = await this.executeSelectPart({ ...sub, where: subWhere });
resolved.$exists = rows.length > 0 !== negate;
continue;
}
// 逻辑组合操作符
if (key === '$and' && Array.isArray(value)) {
resolved.$and = await Promise.all(value.map((sub) => this.resolveSubqueries(sub, contextRow)));
continue;
}
if (key === '$or' && Array.isArray(value)) {
resolved.$or = await Promise.all(value.map((sub) => this.resolveSubqueries(sub, contextRow)));
continue;
}
if (key === '$not' && typeof value === 'object' && value !== null) {
resolved.$not = await this.resolveSubqueries(value, contextRow);
continue;
}
// 字段条件
if (typeof value === 'object' && value !== null) {
resolved[key] = await this.resolveOperatorSubqueries(value);
}
else {
resolved[key] = value;
}
}
return resolved;
}
/**
* 解析操作符值中嵌套的子查询
*/
async resolveOperatorSubqueries(ops) {
const resolved = {};
for (const [op, operand] of Object.entries(ops)) {
// 处理嵌套 $and/$or(在字段级条件中)
if (op === '$and' && Array.isArray(operand)) {
resolved.$and = await Promise.all(operand.map((sub) => this.resolveSubqueries(sub)));
continue;
}
if (op === '$or' && Array.isArray(operand)) {
resolved.$or = await Promise.all(operand.map((sub) => this.resolveSubqueries(sub)));
continue;
}
if (op === '$not') {
resolved.$not = typeof operand === 'object' && operand !== null
? await this.resolveOperatorSubqueries(operand)
: operand;
continue;
}
// 子查询检测
if (typeof operand === 'object' && operand !== null && '$subquery' in operand) {
const subStmt = operand.$subquery;
const subResult = await this.executeSelect(subStmt);
if (op === '$in' || op === '$nin') {
// IN 子查询 → 提取第一列的值列表
const colName = Object.keys(subResult[0] || {})[0];
const values = subResult.map((row) => row[colName]);
resolved[op] = values;
}
else {
// 标量子查询 → 取第一行第一列
if (subResult.length === 0) {
resolved[op] = null;
}
else {
const colName = Object.keys(subResult[0])[0];
resolved[op] = subResult[0][colName];
}
}
}
else {
resolved[op] = operand;
}
}
return resolved;
}
}
/**
* metona-sqlark SQL Parameters — 参数化查询绑定
* @module sql/params
*
* v0.7.0: `db.query(sql, params)` 位置参数(`?`)支持。
* 绑定在词法层面完成:仅替换字符串字面量之外的 `?`,
* 值按 SQL 字面量编码(字符串 `''` 转义、数字/布尔/JSON 直出),
* 从根上规避 SQL 注入(不经过字符串拼接由用户自行转义)。
*
* v0.7.2: 词法扫描感知注释 —— 行注释(`--`)与块注释(slash-star 包裹)中的 `?`
* 与引号不再参与占位符识别与字符串状态机(此前注释中的 `?` 计入占位符导致
* PARAM_ERROR 错位、注释中的单引号触发 "Unterminated string literal")。
*/
/** 将单个参数值编码为 SQL 字面量 */
function encodeParam(value) {
if (value === null || value === undefined)
return 'NULL';
if (typeof value === 'number') {
if (Number.isFinite(value))
return String(value);
return 'NULL'; // NaN/Infinity 无 SQL 字面量 → NULL
}
if (typeof value === 'boolean')
return value ? 'TRUE' : 'FALSE';
if (typeof value === 'string')
return `'${value.replace(/'/g, "''")}'`;
// 对象/数组无 SQL 字面量(SQL 方言不支持 json 字面量),显式拒绝而非静默错配
throw new DatabaseError('Object/array query parameters are not supported by SQL binding (pass JSON strings explicitly)', 'PARAM_ERROR');
}
/**
* 将 SQL 中的位置参数 `?`(字符串字面量与注释之外)替换为编码后的字面量。
* @param sql 含 `?` 占位符的 SQL
* @param params 位置参数数组
* @throws PARAM_ERROR 参数数量不匹配
*/
function bindParameters(sql, params) {
// undefined = 不启用绑定;[] + 含 ? 的 SQL 由循环内报 PARAM_ERROR
if (!params)
return sql;
let out = '';
let i = 0;
let pIdx = 0;
let quote = null;
while (i < sql.length) {
const ch = sql[i];
if (quote !== null) {
out += ch;
if (ch === quote) {
// SQL 标准 '' 转义:两个连续引号 = 一个引号(原样保留)
if (sql[i + 1] === quote) {
out += sql[i + 1];
i += 2;
continue;
}
quote = null;
}
i++;
continue;
}
if (ch === "'" || ch === '"') {
quote = ch;
out += ch;
i++;
continue;
}
// v0.7.2: 行注释 `-- ...`(含其中的 ? 与引号)原样保留、不参与绑定
if (ch === '-' && sql[i + 1] === '-') {
while (i < sql.length && sql[i] !== '\n' && sql[i] !== '\r') {
out += sql[i];
i++;
}
continue;
}
// v0.7.2: 块注释(slash-star 包裹)同样跳过
if (ch === '/' && sql[i + 1] === '*') {
out += sql[i] + sql[i + 1];
i += 2;
while (i < sql.length && !(sql[i] === '*' && sql[i + 1] === '/')) {
out += sql[i];
i++;
}
if (i < sql.length) {
out += sql[i] + sql[i + 1];
i += 2;
}
continue;
}
if (ch === '?') {
if (pIdx >= params.length) {
throw new DatabaseError(`Too few query parameters: placeholder #${pIdx + 1} has no value (got ${params.length} total)`, 'PARAM_ERROR');
}
out += encodeParam(params[pIdx]);
pIdx++;
i++;
continue;
}
out += ch;
i++;
}
if (quote !== null) {
throw new DatabaseError('Unterminated string literal in SQL', 'PARSE_ERROR');
}
if (pIdx < params.length) {
throw new DatabaseError(`Too many query parameters: ${params.length} provided but only ${pIdx} placeholders`, 'PARAM_ERROR');
}
return out;
}
/**
* metona-sqlark Transaction — 事务管理
* @module transaction
*
* v0.1.13: 支持真正的回滚 — 利用引擎层 begin/commit/rollback 实现原子性。
*/
// ---------------------------------------------------------------------------
// Transaction
// ---------------------------------------------------------------------------
class Transaction {
constructor(engine) {
this.tables = new Map();
this.completed = false;
this.engine = engine;
}
/** 获取表操作对象 */
table(tableName) {
let t = this.tables.get(tableName);
if (!t) {
t = new Table(this.engine, tableName);
this.tables.set(tableName, t);
}
return t;
}
/** 标记事务完成(由 TransactionManager 调用) */
_markCompleted() {
this.completed = true;
}
/** 是否已完成 */
isCompleted() {
return this.completed;
}
}
// ---------------------------------------------------------------------------
// TransactionManager
// ---------------------------------------------------------------------------
class TransactionManager {
constructor(engine) {
this.engine = engine;
}
/** 执行事务 — 支持自动回滚 */
async execute(fn) {
const trx = new Transaction(this.engine);
// 开始引擎层事务
await this.engine.beginTransaction();
try {
const result = await fn(trx);
// 成功 → 提交
await this.engine.commitTransaction();
trx._markCompleted();
return result;
}
catch (error) {
// 失败 → 回滚。v0.7.1: 回滚自身失败不掩盖原始事务错误
// (此前 rollback 抛错会替换掉真正导致失败的异常,定位困难)
try {
await this.engine.rollbackTransaction();
}
catch { /* 回滚失败保留原始错误 */ }
if (error instanceof DatabaseError)
throw error;
throw new DatabaseError(`Transaction failed: ${error.message}`, 'TRANSACTION_ERROR', error);
}
}
}
/**
* metona-sqlark Plugin — 插件系统
* @module plugin
*
* 管理插件的注册、生命周期和钩子调度。
*/
// ---------------------------------------------------------------------------
// PluginManager
// ---------------------------------------------------------------------------
class PluginManager {
constructor() {
this.plugins = [];
this.hooks = new Map();
}
/** 注册插件 */
register(plugin, db) {
// 按优先级插入
const priority = plugin.priority ?? 0;
const insertIndex = this.plugins.findIndex((p) => (p.priority ?? 0) < priority);
if (insertIndex === -1) {
this.plugins.push(plugin);
}
else {
this.plugins.splice(insertIndex, 0, plugin);
}
// 安装(传入 db 实例)
plugin.install(db);
}
/** 卸载插件 */
unregister(pluginName) {
const idx = this.plugins.findIndex((p) => p.name === pluginName);
if (idx !== -1) {
this.plugins[idx].destroy();
this.plugins.splice(idx, 1);
}
}
/** 获取所有已注册插件 */
getPlugins() {
return [...this.plugins];
}
/** 添加钩子回调 */
on(hook, callback) {
const callbacks = this.hooks.get(hook) ?? [];
callbacks.push(callback);
this.hooks.set(hook, callbacks);
}
/** 移除钩子回调 */
off(hook, callback) {
const callbacks = this.hooks.get(hook);
if (callbacks) {
const idx = callbacks.indexOf(callback);
if (idx !== -1)
callbacks.splice(idx, 1);
}
}
/** 触发钩子 */
async trigger(hook, ...args) {
const callbacks = this.hooks.get(hook);
if (callbacks) {
for (const cb of callbacks) {
await cb(...args);
}
}
}
/** 销毁所有插件 */
destroy() {
for (const plugin of this.plugins) {
try {
plugin.destroy();
}
catch (e) {
// eslint-disable-next-line no-console
console.warn(`[metona-sqlark] Plugin "${plugin.name}" destroy error:`, e);
}
}
this.plugins = [];
this.hooks.clear();
}
}
/**
* metona-sqlark Core — 数据库主类
* @module core
*
* 管理数据库生命周期、引擎调度、表操作、SQL 查询、事务和插件。
*/
// ---------------------------------------------------------------------------
// MetonaSqlark
// ---------------------------------------------------------------------------
class MetonaSqlark {
/**
* v0.7.1: 静态工厂(与 connect/disconnect 同一入口风格)。
* 此前 create 仅存在于 api 对象 / window 挂载 —— README/站点示例的
* `MetonaSqlark.create({...})` 在 ESM/Node 下是 undefinedTypeError)。
*/
static async create(config) {
const db = new MetonaSqlark(config);
await db.init();
return db;
}
/** 获取版本号 */
get version() { return this._version; }
/** 查询结果行数上限 */
get maxRowsPerQuery() { return this.config.maxRowsPerQuery ?? 0; }
/** 调试模式 */
get debug() { return this.config.debug ?? false; }
constructor(config) {
this.ready = false;
this.tableCache = new Map();
/** 多标签页同步通道(v0.3.2 */
this.channel = null;
// ---- 发布订阅 ----
this.listeners = new Map();
// ---- 迁移 ----
this.migrations = new Map();
this.config = config;
this.name = config.name ?? DB_DEFAULTS.name;
this.mode = config.mode ?? DB_DEFAULTS.mode;
this._version = config.version ?? DB_DEFAULTS.version;
this.pluginManager = new PluginManager();
// v0.3.2: 多标签页同步 — BroadcastChannel 广播表变更
if (config.multiTabSync && typeof BroadcastChannel !== 'undefined') {
this.channel = new BroadcastChannel(`metona-sqlark:${this.name}`);
this.channel.onmessage = (event) => {
const msg = event.data;
if (!msg || msg.type !== 'change')
return;
this.emit(msg.table ?? '', { type: 'external', table: msg.table ?? '' });
// Hybrid 引擎:从磁盘重载内存,保证读到其他标签页的最新数据
if (this.engine instanceof HybridEngine) {
this.engine.reloadMemoryFromDisk().catch(() => {
// 重载失败不影响主流程(下次读可能短暂过期)
});
}
};
}
}
// ---- 初始化 ----
/** 初始化数据库(创建引擎、打开连接) */
async init() {
// 创建引擎
this.engine = this.createEngine();
// 打开连接
await this.engine.open(this.name, this.version);
// v0.4.2-fix (P2-7): 从库内加载持久化的迁移版本,
// 重启后 migrateTo 从持久化版本继续执行,不再每次从 config.version 重置
if (typeof this.engine.getMeta === 'function') {
try {
const persistedVersion = await this.engine.getMeta('__metona_version');
if (persistedVersion != null && Number(persistedVersion) >= 1) {
this._version = Math.max(this._version, Math.floor(Number(persistedVersion)));
}
}
catch { /* 读取失败回退 config.version */ }
}
// 初始化执行器和事务管理器
this.executor = new QueryExecutor(this.engine, this.maxRowsPerQuery);
this.transactionManager = new TransactionManager(this.engine);
// 注册插件
if (this.config.plugins) {
for (const plugin of this.config.plugins) {
this.pluginManager.register(plugin, this);
}
}
this.ready = true;
// 回调
if (this.config.onReady) {
this.config.onReady(this);
}
}
/** 检查是否就绪 */
isReady() {
return this.ready;
}
// ---- 表管理 ----
/** 创建表 */
async defineTable(name, columns) {
this.ensureReady();
const schema = createSchema(name, columns);
try {
await this.pluginManager.trigger('beforeCreateTable', schema);
await this.engine.createTable(schema);
await this.pluginManager.trigger('afterCreateTable', schema);
}
catch (error) {
this._onError(error);
throw error;
}
// 清除缓存
this.tableCache.delete(name);
}
/** 获取表操作对象 */
table(name) {
this.ensureReady();
let t = this.tableCache.get(name);
if (!t) {
// v0.3.2: 表操作写入后广播变更(多标签页同步)
// v0.5.1: CRUD 生命周期钩子真实接线(beforeInsert/afterInsert/...
t = new Table(this.engine, name, this.executor, (tableName) => this.broadcastChange(tableName), (hook, args) => this.pluginManager.trigger(hook, ...args));
this.tableCache.set(name, t);
}
return t;
}
/** 删除表 */
async dropTable(name) {
this.ensureReady();
try {
await this.pluginManager.trigger('beforeDropTable', name);
await this.engine.dropTable(name);
await this.pluginManager.trigger('afterDropTable', name);
}
catch (error) {
this._onError(error);
throw error;
}
this.tableCache.delete(name);
}
/** 获取所有表名 */
async getTableNames() {
this.ensureReady();
return this.engine.getTableNames();
}
// ---- SQL 查询 ----
/**
* 执行 SQL 字符串查询。
* v0.7.0: 支持位置参数(`?`)—— `db.query('SELECT * FROM t WHERE id = ?', ['1'])`。
* 参数按 SQL 字面量安全编码(字符串 '' 转义),杜绝 SQL 注入。
*/
async query(sql, params) {
this.ensureReady();
const startTime = this.debug ? Date.now() : 0;
await this.pluginManager.trigger('beforeQuery', sql);
let result;
try {
// v0.7.0: 参数绑定(仅替换字符串字面量之外的 ?)
const boundSql = bindParameters(sql, params);
// v0.3.0: 支持分号分隔的多语句,逐条顺序执行,返回最后一条的结果
const statements = parseAll(boundSql);
for (const stmt of statements) {
// v0.5.1: SQL 写语句触发 CRUD 生命周期钩子(与 Table API 路径一致)
await this.triggerStatementHooks(stmt, 'before');
result = await this.executor.execute(stmt);
await this.triggerStatementHooks(stmt, 'after', result);
// v0.3.2: 写语句广播表变更(多标签页同步)
const table = this.writeStatementTable(stmt);
if (table)
this.broadcastChange(table);
}
}
catch (error) {
this._onError(error);
throw error;
}
await this.pluginManager.trigger('afterQuery', sql, result);
if (this.debug) {
const elapsed = Date.now() - startTime;
const rows = Array.isArray(result) ? result.length : 0;
this._debug(`query [${elapsed}ms] ${rows} rows: ${sql.slice(0, 100)}`);
}
return result;
}
// ---- 流式查询(v0.4.0 ----
/**
* 流式查询:逐行回调,不一次性物化全部结果(大表友好)。
* 支持简单 SELECTWHERE/LIMIT/OFFSET/列投影);
* JOIN/GROUP BY/UNION/聚合/ORDER BY 自动回退为物化查询后逐行回调。
*
* @example
* ```ts
* let total = 0;
* await db.queryStream('SELECT * FROM logs WHERE level = \'error\'', (row) => {
* total++;
* processRow(row);
* });
* ```
*/
async queryStream(sql, onRow) {
this.ensureReady();
const stmt = parseAll(sql)[0];
if (!stmt || stmt.type !== 'SELECT') {
throw new DatabaseError('queryStream only supports SELECT statements', 'NOT_SUPPORTED');
}
const select = stmt;
// 不可流式场景:JOIN / GROUP BY / HAVING / DISTINCT / 聚合 / UNION / 关联子查询 / ORDER BY
const aggregate = select.columns.some((c) => /^(COUNT|SUM|AVG|MIN|MAX)\(/i.test(c));
const streamable = !select.joins && !select.groupBy && !select.having && !select.distinct
&& !aggregate && !(select.orderBy && select.orderBy.length > 0)
&& !(select.where && select.where['$exists'] !== undefined);
if (streamable && typeof this.engine.findStream === 'function') {
// 用户回调为 async(返回 Promise)时引擎同步扫描无法 await → 回退物化
const isAsync = onRow.constructor?.name === 'AsyncFunction';
if (!isAsync) {
const where = this.normalizeWhereForStream(select);
const plainCols = select.columns.filter((c) => !/\s+AS\s+\w+$/i.test(c));
return this.engine.findStream(select.from, {
table: select.from,
columns: plainCols.length > 0 && plainCols[0] !== '*' ? plainCols : ['*'],
where: where && Object.keys(where).length > 0 ? where : undefined,
limit: select.limit,
offset: select.offset,
}, onRow);
}
}
// 回退:物化后逐行回调
const result = await this.query(sql);
if (Array.isArray(result)) {
for (const row of result) {
await onRow(row);
}
return result.length;
}
return 0;
}
/** 流式查询用:剥离主表别名前缀(复用 query 路径的规范化逻辑) */
normalizeWhereForStream(select) {
const aliases = [select.alias ?? select.from].filter(Boolean);
const strip = (col) => {
for (const a of aliases) {
if (col.startsWith(`${a}.`))
return col.slice(a.length + 1);
}
return col;
};
const walk = (w) => {
const out = {};
for (const [k, v] of Object.entries(w)) {
if (k === '$and' || k === '$or') {
out[k] = v.map(walk);
}
else if (k === '$not' && typeof v === 'object' && v !== null) {
out.$not = walk(v);
}
else {
out[strip(k)] = v;
}
}
return out;
};
return walk(select.where ?? {});
}
// ---- 事务 ----
/** 执行事务 */
async transaction(fn) {
this.ensureReady();
await this.pluginManager.trigger('beforeTransaction');
try {
const result = await this.transactionManager.execute(fn);
await this.pluginManager.trigger('afterTransaction');
return result;
}
catch (error) {
this._onError(error);
throw error;
}
}
// ---- 导入导出 ----
/** 导出表数据为 JSON */
async exportTable(tableName) {
this.ensureReady();
return this.engine.find(tableName, { table: tableName });
}
/** 导入 JSON 数据到表 */
async importTable(tableName, data) {
this.ensureReady();
try {
return await this.engine.insert(tableName, data);
}
catch (error) {
this._onError(error);
throw error;
}
}
/** 导出整个数据库为 JSON */
async exportAll() {
this.ensureReady();
const result = {};
const names = await this.engine.getTableNames();
for (const name of names) {
result[name] = await this.engine.find(name, { table: name });
}
return result;
}
/**
* v0.5.1: 在线备份 — 导出全库一致性快照。
* Aria 引擎走引擎级 backup()(MVCC 一致性视图);其余引擎回退 exportAll()。
*/
async backup() {
this.ensureReady();
if (typeof this.engine.backup === 'function') {
return this.engine.backup();
}
return this.exportAll();
}
/** 订阅表变更 */
subscribe(tableName, callback) {
const key = `change:${tableName}`;
if (!this.listeners.has(key))
this.listeners.set(key, new Set());
this.listeners.get(key).add(callback);
return () => this.listeners.get(key)?.delete(callback);
}
/** 触发变更事件 */
emit(tableName, event) {
const key = `change:${tableName}`;
this.listeners.get(key)?.forEach((cb) => cb(event));
}
// ---- 多标签页同步(v0.3.2 ----
/** 广播表变更到其他标签页(多标签页同步) */
broadcastChange(tableName) {
if (!this.channel)
return;
try {
this.channel.postMessage({ type: 'change', table: tableName });
}
catch {
// 广播失败不影响主流程
}
}
/** 写语句对应的表名(多标签页广播用) */
writeStatementTable(stmt) {
switch (stmt.type) {
case 'INSERT': return stmt.into;
case 'UPDATE': return stmt.table;
case 'DELETE': return stmt.from;
case 'CREATE_TABLE':
case 'DROP_TABLE':
case 'TRUNCATE_TABLE':
return stmt.name;
case 'ALTER_TABLE': return stmt.name;
case 'CREATE_INDEX':
case 'DROP_INDEX':
return stmt.table;
default:
return null;
}
}
/**
* v0.5.1: SQL 写语句触发 CRUD 生命周期钩子。
* INSERT/UPDATE/DELETE 分别触发 beforeInsert/afterInsert、beforeUpdate/afterUpdate、
* beforeDelete/afterDelete(参数与 Table API 路径一致)。
*/
async triggerStatementHooks(stmt, phase, result) {
switch (stmt.type) {
case 'INSERT': {
const rows = (stmt.values ?? []).map((vals) => {
const row = {};
const cols = stmt.columns ?? [];
for (let i = 0; i < vals.length; i++) {
row[cols[i] ?? String(i)] = vals[i];
}
return row;
});
if (phase === 'before')
await this.pluginManager.trigger('beforeInsert', rows);
else
await this.pluginManager.trigger('afterInsert', rows, result);
break;
}
case 'UPDATE': {
const query = { table: stmt.table, where: stmt.where };
if (phase === 'before')
await this.pluginManager.trigger('beforeUpdate', query, stmt.sets);
else
await this.pluginManager.trigger('afterUpdate', query, stmt.sets, result);
break;
}
case 'DELETE': {
const query = { table: stmt.from, where: stmt.where };
if (phase === 'before')
await this.pluginManager.trigger('beforeDelete', query);
else
await this.pluginManager.trigger('afterDelete', query, result);
break;
}
}
}
/** 注册迁移 */
addMigration(version, up) {
this.migrations.set(version, up);
}
/** 执行迁移到指定版本 */
async migrateTo(targetVersion) {
this.ensureReady();
for (const [version, up] of [...this.migrations.entries()].sort((a, b) => a[0] - b[0])) {
if (version <= targetVersion && version > this._version) {
await up(this);
this._version = version;
}
}
// v0.4.2-fix (P2-7): 迁移版本持久化到库内,重启后从持久化版本继续,
// 避免"version 重置导致已执行迁移重跑(不幂等就炸)"或"版本门槛跳过迁移"
if (typeof this.engine.setMeta === 'function') {
try {
await this.engine.setMeta('__metona_version', String(this._version));
}
catch { /* 持久化失败不阻塞迁移流程 */ }
}
}
// ---- 自愈 / 重置(v0.4.2-fix, P2-9 ----
/**
* 崩溃恢复自愈 — 校验并清理损坏数据、恢复一致性。
* 检测到异常后调用,无需删库重建。
*/
async repair() {
this.ensureReady();
if (typeof this.engine.repair === 'function') {
await this.engine.repair();
this.tableCache.clear();
return;
}
// 兜底:重建表缓存
this.tableCache.clear();
}
/**
* 清空全部数据与表结构(保留库本身)。
* 支持后续继续使用本实例重建表。
*/
async clearAll() {
this.ensureReady();
if (typeof this.engine.clearAll === 'function') {
await this.engine.clearAll();
}
else {
const names = await this.engine.getTableNames();
for (const name of names) {
await this.engine.dropTable(name);
}
}
this.tableCache.clear();
}
// ---- 插件 ----
/** 获取插件管理器 */
getPluginManager() {
return this.pluginManager;
}
/** 注册钩子 */
on(hook, callback) {
this.pluginManager.on(hook, callback);
}
// ---- 生命周期 ----
/** 关闭数据库 */
async close() {
if (this.channel) {
this.channel.close();
this.channel = null;
}
this.pluginManager.destroy();
// v0.7.1: init 失败/未调用时 close 不应崩溃(此前 this.engine undefined → TypeError
if (this.engine) {
await this.engine.close();
}
this.tableCache.clear();
this.ready = false;
}
/** 获取底层引擎 */
getEngine() {
return this.engine;
}
// ---- 内部 ----
createEngine() {
const mode = this.mode;
const diskEngine = this.config.diskEngine ?? 'opfs';
switch (mode) {
case 'memory':
return new MemoryEngine();
case 'disk':
// v0.6.0: 自研 KVStoreEngine(完全移除 IndexedDB
return new KVStoreEngine();
case 'aria':
// v0.4.5: 透传 AriaEngine 专属配置(walSyncMode/checkpointInterval/encryption/pageStorage 等)
// v0.6.1: diskEngine 'kv' → 自研 KVStore 后端
return new AriaEngine({
storageBackend: diskEngine === 'memory' ? 'memory' : diskEngine === 'kv' ? 'kv' : 'opfs',
...(this.config.aria ?? {}),
});
case 'hybrid':
return new HybridEngine(diskEngine);
default:
throw new DatabaseError(`Unknown storage mode: ${mode}`, 'CONFIG_ERROR');
}
}
ensureReady() {
if (!this.ready) {
throw new DatabaseError('Database not initialized. Call await db.init() first.', 'DB_NOT_READY');
}
}
/** 错误回调分发 */
_onError(error) {
if (this.config.onError) {
try {
this.config.onError(error);
}
catch { /* 避免回调自身异常影响主流程 */ }
}
}
/** 调试日志 */
_debug(msg, ...args) {
if (this.debug) {
// eslint-disable-next-line no-console
console.debug(`[MetonaSqlark:${this.name}] ${msg}`, ...args);
}
}
}
/**
* metona-sqlark Connection Manager — 数据库实例连接池
* @module connection-manager
*
* v0.1.13: 避免重复创建同名数据库实例,通过 connect() 复用已有连接。
* 管理实例生命周期,防止重复 open IndexedDB。
*/
// ---------------------------------------------------------------------------
// ConnectionManager
// ---------------------------------------------------------------------------
class ConnectionManager {
constructor() {
/** 活跃连接:dbName → MetonaSqlark */
this.connections = new Map();
/** 连接引用计数:dbName → count */
this.refCount = new Map();
}
/**
* 获取或创建数据库实例
*
* 如果同名数据库已打开,复用已有实例并增加引用计数。
* 否则创建新实例。
*
* @example
* ```ts
* const db = await MetonaSqlark.connect({ name: 'my-app', mode: 'hybrid' });
* // ... use db
* await db.disconnect(); // 引用计数 -1,归零时自动关闭
* ```
*/
async connect(config) {
const name = config.name;
// 已有连接 → 复用
const existing = this.connections.get(name);
if (existing && existing.isReady()) {
const count = (this.refCount.get(name) ?? 0) + 1;
this.refCount.set(name, count);
return existing;
}
// 创建新连接
const db = new MetonaSqlark(config);
await db.init();
this.connections.set(name, db);
this.refCount.set(name, 1);
// 注入 disconnect 方法
db.disconnect = async () => {
await this.release(name);
};
return db;
}
/**
* 释放连接引用。引用计数归零时自动关闭数据库。
*/
async release(dbName) {
const count = (this.refCount.get(dbName) ?? 1) - 1;
if (count <= 0) {
const db = this.connections.get(dbName);
if (db) {
await db.close();
this.connections.delete(dbName);
}
this.refCount.delete(dbName);
}
else {
this.refCount.set(dbName, count);
}
}
/**
* 强制关闭指定数据库(忽略引用计数)
*/
async forceClose(dbName) {
const db = this.connections.get(dbName);
if (db) {
await db.close();
this.connections.delete(dbName);
}
this.refCount.delete(dbName);
}
/**
* 强制关闭所有连接
*/
async closeAll() {
for (const [, db] of this.connections) {
try {
await db.close();
}
catch { /* ignore */ }
}
this.connections.clear();
this.refCount.clear();
}
/**
* 获取所有活跃连接名
*/
getActiveConnections() {
return Array.from(this.connections.keys());
}
}
// ---------------------------------------------------------------------------
// 全局单例
// ---------------------------------------------------------------------------
const manager = new ConnectionManager();
// 挂载到 MetonaSqlark 静态方法(通过 any 绕过 TS 类型检查)
const M = MetonaSqlark;
M.connect = (config) => manager.connect(config);
M.disconnect = (dbName) => manager.release(dbName);
M.disconnectAll = () => manager.closeAll();
M.getActiveConnections = () => manager.getActiveConnections();
/**
* metona-sqlark — 入口文件
* @module metona-sqlark
* @version 0.4.1
*
* 前端关系型数据库,内存与磁盘双模式。
* 支持 Query Builder 链式 API 和 SQL 字符串查询。
*/
// ---------------------------------------------------------------------------
// 工厂函数
// ---------------------------------------------------------------------------
/**
* 创建数据库实例并初始化
*
* @example
* ```ts
* const db = await MetonaSqlark.create({
* name: 'my-app',
* mode: 'hybrid',
* });
*
* await db.defineTable('users', {
* id: { type: 'string', primaryKey: true },
* name: { type: 'string', required: true },
* });
*
* await db.table('users').insert({ id: '1', name: 'Alice' });
* const results = await db.query('SELECT * FROM users');
* ```
*/
async function create(config) {
// v0.7.1: 委托静态 create(类与独立函数行为一致,README 两种写法均可)
return MetonaSqlark.create(config);
}
// ---------------------------------------------------------------------------
// 全局 API
// ---------------------------------------------------------------------------
const api = {
VERSION,
version: VERSION,
create,
MetonaSqlark,
MeSqlark: MetonaSqlark,
};
if (typeof window !== 'undefined') {
window.MetonaSqlark = api;
window.MeSqlark = api;
}
// 别名
const MeSqlark = MetonaSqlark;
export { AriaEngine, HybridEngine, KVStoreEngine, MeSqlark, MemoryEngine, MetonaSqlark, OPFSBackend, Table, VERSION, api, create, api as default, parse, parseAll, tokenize };
//# sourceMappingURL=metona-sqlark.esm.js.map