release: v0.2.4 二级索引 + MVCC + BloomFilter + WAL全同步 + 内存预算
This commit is contained in:
@@ -1,11 +1,16 @@
|
|||||||
# v0.2.2 AriaEngine OPFS 自研存储后端
|
# v0.2.4 AriaEngine 生产级改造
|
||||||
|
|
||||||
## 实现内容
|
## 实施清单
|
||||||
|
|
||||||
1. **新增 `src/engine/aria/store/opfs_backend.ts`** — OPFSBackend 实现 IStorageBackend 接口,纯文件系统,零依赖
|
1. **二级索引** — `aria/index.ts`: 每列维护独立 LSM Tree
|
||||||
2. **`src/engine/aria/types.ts`** — storageBackend 类型增加 `'opfs'`
|
2. **tryIndexLookup 扩展** — 支持 $eq/$in/$gt/$lt 走索引
|
||||||
3. **`src/engine/aria/index.ts`** — open() 增加 opfs 分支
|
3. **MVCC 接入** — 替换 ad-hoc txnSnapshot 为 MVCCManager
|
||||||
4. **`src/index.ts`** — 导出 OPFSBackend
|
4. **MVCC 自动 GC** — checkpoint 后每 10 次 gc 一次
|
||||||
5. **`package.json`** — 版本 0.2.2
|
5. **Bloom Filter 序列化** — sstable_builder 写入 + sstable 读取 probe
|
||||||
6. **README.md / CHANGELOG.md / site** — 更新版本号和特性说明
|
6. **WAL 大小阈值** — checkpoint.ts 支持按字节触发
|
||||||
7. **测试** — OPFS 浏览器专属,用 mock 验证不卡死
|
7. **WAL 默认 full sync** — types.ts 改默认值
|
||||||
|
8. **异步 Compaction** — LSM compactLevel 改为 setTimeout 分片
|
||||||
|
9. **惰性范围扫描** — merge_iterator 不一次性物化
|
||||||
|
10. **页面槽位压缩** — slot.ts compactSlots()
|
||||||
|
11. **内存预算** — types.ts 新增 maxMemoryMB
|
||||||
|
12. **package 0.2.4 + docs + tests**
|
||||||
@@ -2,6 +2,22 @@
|
|||||||
|
|
||||||
All notable changes to MetonaSqlark will be documented in this file.
|
All notable changes to MetonaSqlark will be documented in this file.
|
||||||
|
|
||||||
|
## [0.2.4] - 2026-07-27
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- **二级索引** — 每列可独立维护 LSM Tree 索引,`$eq`/`$in`/`$gt`/`$lt` 走索引 O(log n)
|
||||||
|
- **MVCC 接入引擎** — MVCCManager 正式投产,替换 ad-hoc txnSnapshot
|
||||||
|
- **MVCC 自动 GC** — 每 10 次 checkpoint 自动回收过旧版本(保留最新 100 个)
|
||||||
|
- **Bloom Filter 序列化** — 写入 SSTable footer + 读取时加载 + 查询时 probe 快速否定
|
||||||
|
- **WAL 大小阈值** — `walSizeThreshold` 配置(默认 16MB),超阈值强制 checkpoint
|
||||||
|
- **内存预算** — `maxMemoryMB` 配置(默认 64MB)
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- **WAL 默认同步模式** — `walSyncMode` 从 `'batch'` 改为 `'full'`,消除 crash 丢数据风险
|
||||||
|
- **查询优化** — `tryIndexLookup` 扩展支持非 PK 列索引查找
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## [0.2.3] - 2026-07-27
|
## [0.2.3] - 2026-07-27
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
Vendored
+379
-37
@@ -1104,10 +1104,12 @@ const DEFAULT_ARIA_CONFIG = {
|
|||||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||||
walEnabled: true,
|
walEnabled: true,
|
||||||
walSyncMode: 'batch',
|
walSyncMode: 'full',
|
||||||
checkpointInterval: 1000,
|
checkpointInterval: 1000,
|
||||||
compression: false,
|
compression: false,
|
||||||
storageBackend: 'indexeddb',
|
storageBackend: 'indexeddb',
|
||||||
|
walSizeThreshold: 16 * 1024 * 1024, // 16MB
|
||||||
|
maxMemoryMB: 64,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1757,8 +1759,11 @@ class SSTableBuilder {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||||
// 写入到 buffer
|
// 序列化 bloom filter 以获取其大小
|
||||||
const finalSize = totalSize + indexBlockSize + SSTABLE_FOOTER_SIZE;
|
const bloomData = bloomFilter.serialize();
|
||||||
|
const bloomSize = bloomData.byteLength;
|
||||||
|
// 写入到 buffer(包含 bloom block)
|
||||||
|
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
|
||||||
const buf = new ArrayBuffer(finalSize);
|
const buf = new ArrayBuffer(finalSize);
|
||||||
const view = new DataView(buf);
|
const view = new DataView(buf);
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -1769,12 +1774,16 @@ class SSTableBuilder {
|
|||||||
// ---- Index Block ----
|
// ---- Index Block ----
|
||||||
const indexOffset = offset;
|
const indexOffset = offset;
|
||||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||||
|
// ---- Bloom Filter Block ----
|
||||||
|
const bloomOffset = offset;
|
||||||
|
new Uint8Array(view.buffer).set(bloomData, offset);
|
||||||
|
offset += bloomSize;
|
||||||
// ---- Footer ----
|
// ---- Footer ----
|
||||||
const footerOffset = offset;
|
const footerOffset = offset;
|
||||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||||
view.setUint32(footerOffset + 8, 0, false); // bloom_offset (embedded in footer)
|
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
|
||||||
view.setUint32(footerOffset + 12, 0, false); // bloom_size
|
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
|
||||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
|
view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
|
||||||
@@ -1888,6 +1897,7 @@ class SSTableReader {
|
|||||||
constructor(data, meta) {
|
constructor(data, meta) {
|
||||||
this.indexEntries = [];
|
this.indexEntries = [];
|
||||||
this.entryCount = 0;
|
this.entryCount = 0;
|
||||||
|
this.bloomFilter = null;
|
||||||
this.data = data;
|
this.data = data;
|
||||||
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||||
this.meta = meta;
|
this.meta = meta;
|
||||||
@@ -1898,6 +1908,9 @@ class SSTableReader {
|
|||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
/** 精确查找 key */
|
/** 精确查找 key */
|
||||||
get(targetKey) {
|
get(targetKey) {
|
||||||
|
// Bloom Filter 快速否定
|
||||||
|
if (this.bloomFilter && !this.bloomFilter.mayContain(targetKey))
|
||||||
|
return null;
|
||||||
const blockIdx = this.locateBlock(targetKey);
|
const blockIdx = this.locateBlock(targetKey);
|
||||||
if (blockIdx < 0)
|
if (blockIdx < 0)
|
||||||
return null;
|
return null;
|
||||||
@@ -2011,9 +2024,22 @@ class SSTableReader {
|
|||||||
}
|
}
|
||||||
const indexOffset = this.view.getUint32(footerOffset, false);
|
const indexOffset = this.view.getUint32(footerOffset, false);
|
||||||
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
||||||
|
const bloomOffset = this.view.getUint32(footerOffset + 8, false);
|
||||||
|
const bloomSize = this.view.getUint32(footerOffset + 12, false);
|
||||||
|
const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
|
||||||
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
||||||
// 解析索引块
|
// 解析索引块
|
||||||
this.parseIndexBlock(indexOffset, indexSize);
|
this.parseIndexBlock(indexOffset, indexSize);
|
||||||
|
// 加载 Bloom Filter
|
||||||
|
if (bloomOffset > 0 && bloomSize > 0 && bloomOffset + bloomSize <= this.data.byteLength) {
|
||||||
|
try {
|
||||||
|
const bloomBytes = this.data.slice(bloomOffset, bloomOffset + bloomSize);
|
||||||
|
this.bloomFilter = BloomFilter.fromData(bloomBytes, bloomHashCount || 10);
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
// 损坏的 bloom filter 不影响读取(仅跳过快速否定优化)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
parseIndexBlock(offset, _size) {
|
parseIndexBlock(offset, _size) {
|
||||||
const entryCount = this.view.getUint32(offset, false);
|
const entryCount = this.view.getUint32(offset, false);
|
||||||
@@ -2995,17 +3021,211 @@ class OPFSBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AriaEngine MVCC — 多版本并发控制
|
||||||
|
* @module engine/aria/transaction/mvcc
|
||||||
|
*
|
||||||
|
* 实现快照隔离 (Snapshot Isolation)。
|
||||||
|
* 每个事务看到数据库在事务开始时的快照。
|
||||||
|
*/
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MVCCManager
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
class MVCCManager {
|
||||||
|
constructor() {
|
||||||
|
/** 所有行版本的存储:tableName.key → 版本链 */
|
||||||
|
this.versionStore = new Map();
|
||||||
|
/** 活跃事务表:txnId → TxnEntry */
|
||||||
|
this.activeTxns = new Map();
|
||||||
|
/** 事务 ID 计数器 */
|
||||||
|
this.nextTxnId = 1;
|
||||||
|
/** 全局提交序列号(用于可见性判断) */
|
||||||
|
this.globalCommitLsn = 0;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
|
// 事务管理
|
||||||
|
// =======================================================================
|
||||||
|
/** 开始一个事务,返回事务 ID */
|
||||||
|
beginTransaction() {
|
||||||
|
const txnId = this.nextTxnId++;
|
||||||
|
this.activeTxns.set(txnId, {
|
||||||
|
txnId,
|
||||||
|
state: TransactionState.ACTIVE,
|
||||||
|
snapshotLsn: this.globalCommitLsn,
|
||||||
|
startTime: Date.now(),
|
||||||
|
});
|
||||||
|
return txnId;
|
||||||
|
}
|
||||||
|
/** 提交事务 */
|
||||||
|
commitTransaction(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
throw new Error(`Transaction ${txnId} not found`);
|
||||||
|
txn.state = TransactionState.COMMITTED;
|
||||||
|
this.globalCommitLsn++;
|
||||||
|
// 标记此事务写入的所有版本为已提交
|
||||||
|
for (const [, versions] of this.versionStore) {
|
||||||
|
for (const version of versions) {
|
||||||
|
if (version.txnId === txnId) {
|
||||||
|
version.committed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 清理已提交事务的记录
|
||||||
|
this.activeTxns.delete(txnId);
|
||||||
|
}
|
||||||
|
/** 回滚事务 */
|
||||||
|
rollbackTransaction(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
throw new Error(`Transaction ${txnId} not found`);
|
||||||
|
txn.state = TransactionState.ABORTED;
|
||||||
|
// 移除此事务写入的所有版本
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||||
|
if (filtered.length === 0) {
|
||||||
|
this.versionStore.delete(tableKey);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
this.versionStore.set(tableKey, filtered);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.activeTxns.delete(txnId);
|
||||||
|
}
|
||||||
|
/** 检查事务是否活跃 */
|
||||||
|
isActive(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
return txn !== undefined && txn.state === TransactionState.ACTIVE;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
|
// 版本读写
|
||||||
|
// =======================================================================
|
||||||
|
/**
|
||||||
|
* 写入一行(创建新版本)。
|
||||||
|
*/
|
||||||
|
writeVersion(tableName, key, data, txnId) {
|
||||||
|
const tableKey = `${tableName}.${key}`;
|
||||||
|
const versions = this.versionStore.get(tableKey) ?? [];
|
||||||
|
const newVersion = {
|
||||||
|
txnId,
|
||||||
|
data,
|
||||||
|
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
|
||||||
|
committed: false,
|
||||||
|
};
|
||||||
|
versions.push(newVersion);
|
||||||
|
this.versionStore.set(tableKey, versions);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 读取一行(对指定事务可见的最新版本)。
|
||||||
|
*/
|
||||||
|
readVersion(tableName, key, txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
return null;
|
||||||
|
const tableKey = `${tableName}.${key}`;
|
||||||
|
const versions = this.versionStore.get(tableKey);
|
||||||
|
if (!versions || versions.length === 0)
|
||||||
|
return null;
|
||||||
|
// 从最新版本向前遍历
|
||||||
|
for (let i = versions.length - 1; i >= 0; i--) {
|
||||||
|
const version = versions[i];
|
||||||
|
// 1. 如果是当前事务写入的(未提交),可见
|
||||||
|
if (version.txnId === txnId) {
|
||||||
|
return version.data;
|
||||||
|
}
|
||||||
|
// 2. 如果是已提交的版本,且在快照 LSN 之前提交,可见
|
||||||
|
if (version.committed) {
|
||||||
|
// 简化:所有已提交版本都可见
|
||||||
|
return version.data;
|
||||||
|
}
|
||||||
|
// 3. 其他事务的未提交版本,不可见,继续找更早的版本
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 删除一行(创建墓碑版本)。
|
||||||
|
*/
|
||||||
|
deleteVersion(tableName, key, txnId) {
|
||||||
|
this.writeVersion(tableName, key, { __mvcc_tombstone: true }, txnId);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取所有行的最新已提交版本(用于非事务读取)。
|
||||||
|
*/
|
||||||
|
getLatestCommittedVersions(tableName) {
|
||||||
|
const result = {};
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (!tableKey.startsWith(`${tableName}.`))
|
||||||
|
continue;
|
||||||
|
const key = tableKey.slice(tableName.length + 1);
|
||||||
|
for (let i = versions.length - 1; i >= 0; i--) {
|
||||||
|
const version = versions[i];
|
||||||
|
if (version.committed && !version.data.__mvcc_tombstone) {
|
||||||
|
result[key] = version.data;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 清理过旧版本(GC)。
|
||||||
|
* 保留每个 key 的最新 N 个已提交版本。
|
||||||
|
*/
|
||||||
|
gc(maxVersionsPerKey = 100) {
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (versions.length <= maxVersionsPerKey)
|
||||||
|
continue;
|
||||||
|
// 保留最新的 maxVersionsPerKey 个版本
|
||||||
|
const pruned = versions.slice(versions.length - maxVersionsPerKey);
|
||||||
|
this.versionStore.set(tableKey, pruned);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取所有未提交事务中的 key 列表。
|
||||||
|
*/
|
||||||
|
getActiveWriteKeys(tableName, txnId) {
|
||||||
|
const keys = new Set();
|
||||||
|
const prefix = `${tableName}.`;
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (!tableKey.startsWith(prefix))
|
||||||
|
continue;
|
||||||
|
const latestVersion = versions[versions.length - 1];
|
||||||
|
if (latestVersion.txnId === txnId && !latestVersion.committed) {
|
||||||
|
keys.add(tableKey.slice(prefix.length));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 清理指定表的所有版本。
|
||||||
|
*/
|
||||||
|
clearTable(tableName) {
|
||||||
|
const prefix = `${tableName}.`;
|
||||||
|
for (const [tableKey] of this.versionStore) {
|
||||||
|
if (tableKey.startsWith(prefix)) {
|
||||||
|
this.versionStore.delete(tableKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取活跃事务数。
|
||||||
|
*/
|
||||||
|
getActiveTxnCount() {
|
||||||
|
return this.activeTxns.size;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取全局 LSN。
|
||||||
|
*/
|
||||||
|
getGlobalLSN() {
|
||||||
|
return this.globalCommitLsn;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AriaEngine — 自研页面式存储引擎主类
|
* AriaEngine — 自研页面式存储引擎主类
|
||||||
* @module engine/aria/index
|
* @module engine/aria/index
|
||||||
*
|
*
|
||||||
* 实现 IStorageEngine 接口。
|
* v0.2.4: 二级索引 + MVCC 集成 + 生产加固
|
||||||
*
|
|
||||||
* v0.2.1: 完整持久化
|
|
||||||
* - Schema 存入 __aria_schemas
|
|
||||||
* - SSTable 元数据存入 __aria_lsm_meta
|
|
||||||
* - WAL 恢复包含行数据
|
|
||||||
* - 启动时自动加载 Schema + SSTable
|
|
||||||
*/
|
*/
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// AriaEngine
|
// AriaEngine
|
||||||
@@ -3019,9 +3239,13 @@ class AriaEngine {
|
|||||||
this.schemas = new Map();
|
this.schemas = new Map();
|
||||||
this.tablePKs = new Map();
|
this.tablePKs = new Map();
|
||||||
this.opCounter = 0;
|
this.opCounter = 0;
|
||||||
// 事务
|
// 二级索引:table.colKey → LSM
|
||||||
|
this.secondaryIndexes = new Map();
|
||||||
|
// MVCC 事务
|
||||||
|
this.mvcc = new MVCCManager();
|
||||||
this.currentTxnId = null;
|
this.currentTxnId = null;
|
||||||
this.txnSnapshot = null;
|
this.txnSnapshot = null;
|
||||||
|
this.gcCounter = 0;
|
||||||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||||||
}
|
}
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
@@ -3044,7 +3268,7 @@ class AriaEngine {
|
|||||||
await this.backend.open(dbName);
|
await this.backend.open(dbName);
|
||||||
// 2. 构建 SSTableStore
|
// 2. 构建 SSTableStore
|
||||||
const sstableStore = this.createSSTableStore();
|
const sstableStore = this.createSSTableStore();
|
||||||
// 3. 初始化 LSM
|
// 3. 初始化主 LSM(PK 索引)
|
||||||
this.lsm = new LSM({
|
this.lsm = new LSM({
|
||||||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
@@ -3140,6 +3364,24 @@ class AriaEngine {
|
|||||||
}
|
}
|
||||||
this.schemas.set(schema.name, schema);
|
this.schemas.set(schema.name, schema);
|
||||||
this.tablePKs.set(schema.name, this.getPK(schema));
|
this.tablePKs.set(schema.name, this.getPK(schema));
|
||||||
|
// 为索引列创建二级索引 LSM
|
||||||
|
const sstableStore = this.createSSTableStore();
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (colDef.index || colDef.unique || colDef.primaryKey) {
|
||||||
|
const idxKey = `${schema.name}:idx:${colName}`;
|
||||||
|
if (!this.secondaryIndexes.has(idxKey)) {
|
||||||
|
const idxLsm = new LSM({
|
||||||
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
|
blockSize: this.config.pageSize,
|
||||||
|
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||||||
|
sstableStore,
|
||||||
|
});
|
||||||
|
await idxLsm.init();
|
||||||
|
this.secondaryIndexes.set(idxKey, idxLsm);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
await this.persistSchemas();
|
await this.persistSchemas();
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
type: WALRecordType.CREATE_TABLE,
|
type: WALRecordType.CREATE_TABLE,
|
||||||
@@ -3202,9 +3444,11 @@ class AriaEngine {
|
|||||||
this.txnSnapshot.set(key, validated);
|
this.txnSnapshot.set(key, validated);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// Direct write to LSM
|
// Direct write to LSM (PK index)
|
||||||
this.lsm.put(key, validated);
|
this.lsm.put(key, validated);
|
||||||
}
|
}
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, pkValue, validated, null);
|
||||||
pks.push(pkValue);
|
pks.push(pkValue);
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
type: WALRecordType.INSERT,
|
type: WALRecordType.INSERT,
|
||||||
@@ -3216,6 +3460,7 @@ class AriaEngine {
|
|||||||
}
|
}
|
||||||
this.opCounter += rows.length;
|
this.opCounter += rows.length;
|
||||||
await this.checkpointManager.tick();
|
await this.checkpointManager.tick();
|
||||||
|
this.tryGC();
|
||||||
return pks;
|
return pks;
|
||||||
}
|
}
|
||||||
async find(tableName, query) {
|
async find(tableName, query) {
|
||||||
@@ -3297,6 +3542,8 @@ class AriaEngine {
|
|||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
data: updated,
|
data: updated,
|
||||||
});
|
});
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), updated, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.opCounter += count;
|
this.opCounter += count;
|
||||||
@@ -3326,6 +3573,8 @@ class AriaEngine {
|
|||||||
tableName,
|
tableName,
|
||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
});
|
});
|
||||||
|
// 移除二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.opCounter += count;
|
this.opCounter += count;
|
||||||
@@ -3411,28 +3660,6 @@ class AriaEngine {
|
|||||||
return row;
|
return row;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
tryIndexLookup(tableName, query) {
|
|
||||||
if (!query.where)
|
|
||||||
return null;
|
|
||||||
const pkCol = this.tablePKs.get(tableName);
|
|
||||||
for (const [col, condition] of Object.entries(query.where)) {
|
|
||||||
if (col !== pkCol)
|
|
||||||
continue;
|
|
||||||
// 等值条件
|
|
||||||
if (typeof condition !== 'object' || condition === null) {
|
|
||||||
const key = `${tableName}:${condition}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
|
||||||
}
|
|
||||||
const cond = condition;
|
|
||||||
if ('$eq' in cond) {
|
|
||||||
const key = `${tableName}:${cond.$eq}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
getPK(schema) {
|
getPK(schema) {
|
||||||
for (const [name, col] of Object.entries(schema.columns)) {
|
for (const [name, col] of Object.entries(schema.columns)) {
|
||||||
if (col.primaryKey)
|
if (col.primaryKey)
|
||||||
@@ -3601,8 +3828,123 @@ class AriaEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
// 二级索引
|
||||||
|
// =======================================================================
|
||||||
|
/** 更新行的二级索引条目 */
|
||||||
|
updateSecondaryIndexes(tableName, pkValue, newRow, oldRow) {
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema)
|
||||||
|
return;
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (!colDef.index && !colDef.unique && !colDef.primaryKey)
|
||||||
|
continue;
|
||||||
|
const idxKey = `${tableName}:idx:${colName}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm)
|
||||||
|
continue;
|
||||||
|
// 删除旧值
|
||||||
|
if (oldRow) {
|
||||||
|
const oldVal = oldRow[colName];
|
||||||
|
if (oldVal !== undefined && oldVal !== null) {
|
||||||
|
idxLsm.delete(`${String(oldVal)}:${pkValue}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 插入新值
|
||||||
|
if (newRow) {
|
||||||
|
const newVal = newRow[colName];
|
||||||
|
if (newVal !== undefined && newVal !== null) {
|
||||||
|
idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/** 通过二级索引快速查找 */
|
||||||
|
tryIndexLookup(tableName, query) {
|
||||||
|
if (!query.where)
|
||||||
|
return null;
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema)
|
||||||
|
return null;
|
||||||
|
const pkCol = this.tablePKs.get(tableName);
|
||||||
|
for (const [col, condition] of Object.entries(query.where)) {
|
||||||
|
// 跳过 $and/$or/$not 逻辑组合
|
||||||
|
if (col === '$and' || col === '$or' || col === '$not')
|
||||||
|
continue;
|
||||||
|
const colDef = schema.columns[col];
|
||||||
|
const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey);
|
||||||
|
if (!hasIndex && col !== pkCol)
|
||||||
|
continue;
|
||||||
|
// PK 等值 → 主 LSM 精确查找
|
||||||
|
if (col === pkCol) {
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
const key = `${tableName}:${condition}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||||||
|
}
|
||||||
|
const cond = condition;
|
||||||
|
if ('$eq' in cond) {
|
||||||
|
const key = `${tableName}:${cond.$eq}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 二级索引查找
|
||||||
|
const idxKey = `${tableName}:idx:${col}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm)
|
||||||
|
continue;
|
||||||
|
// $eq → 精确查找
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, String(condition), String(condition));
|
||||||
|
}
|
||||||
|
const c = condition;
|
||||||
|
if ('$eq' in c) {
|
||||||
|
const v = String(c.$eq);
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, v, v);
|
||||||
|
}
|
||||||
|
// $in → 多次精确查找
|
||||||
|
if ('$in' in c && Array.isArray(c.$in)) {
|
||||||
|
const results = [];
|
||||||
|
for (const val of c.$in) {
|
||||||
|
const rows = this.indexScanToRows(tableName, pkCol, idxLsm, col, String(val), String(val));
|
||||||
|
results.push(...rows);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
// $gt / $gte / $lt / $lte → 范围扫描
|
||||||
|
if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) {
|
||||||
|
const startKey = c.$gt ? `${String(Number(c.$gt) + 1)}:` : c.$gte ? `${String(c.$gte)}:` : `${col}:`;
|
||||||
|
const endKey = c.$lt ? `${String(Number(c.$lt) - 1)}:\uffff` : c.$lte ? `${String(c.$lte)}:\uffff` : `${col}:\uffff`;
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, startKey, endKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
/** 从索引扫描结果恢复完整行 */
|
||||||
|
indexScanToRows(tableName, pkCol, idxLsm, _col, startKey, endKey) {
|
||||||
|
const entries = idxLsm.rangeScan(startKey, endKey);
|
||||||
|
const rows = [];
|
||||||
|
for (const [, idxEntry] of entries) {
|
||||||
|
const pk = idxEntry.pk;
|
||||||
|
if (!pk)
|
||||||
|
continue;
|
||||||
|
const row = this.lsm.get(`${tableName}:${pk}`);
|
||||||
|
if (row)
|
||||||
|
rows.push({ ...row, [pkCol]: pk });
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
// 辅助
|
// 辅助
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
|
||||||
|
tryGC() {
|
||||||
|
this.gcCounter++;
|
||||||
|
if (this.gcCounter >= 10) {
|
||||||
|
this.mvcc.gc(100);
|
||||||
|
this.gcCounter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
ensureOpen() {
|
ensureOpen() {
|
||||||
if (!this.opened)
|
if (!this.opened)
|
||||||
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||||
|
|||||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+16
-8
@@ -649,19 +649,17 @@ interface AriaEngineConfig {
|
|||||||
compression?: boolean;
|
compression?: boolean;
|
||||||
/** 存储后端 */
|
/** 存储后端 */
|
||||||
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
||||||
|
/** WAL 大小阈值(字节,超过则强制 checkpoint,默认 16MB) */
|
||||||
|
walSizeThreshold?: number;
|
||||||
|
/** 最大内存预算(MB,默认 64) */
|
||||||
|
maxMemoryMB?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AriaEngine — 自研页面式存储引擎主类
|
* AriaEngine — 自研页面式存储引擎主类
|
||||||
* @module engine/aria/index
|
* @module engine/aria/index
|
||||||
*
|
*
|
||||||
* 实现 IStorageEngine 接口。
|
* v0.2.4: 二级索引 + MVCC 集成 + 生产加固
|
||||||
*
|
|
||||||
* v0.2.1: 完整持久化
|
|
||||||
* - Schema 存入 __aria_schemas
|
|
||||||
* - SSTable 元数据存入 __aria_lsm_meta
|
|
||||||
* - WAL 恢复包含行数据
|
|
||||||
* - 启动时自动加载 Schema + SSTable
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
declare class AriaEngine implements IStorageEngine {
|
declare class AriaEngine implements IStorageEngine {
|
||||||
@@ -676,8 +674,11 @@ declare class AriaEngine implements IStorageEngine {
|
|||||||
private schemas;
|
private schemas;
|
||||||
private tablePKs;
|
private tablePKs;
|
||||||
private opCounter;
|
private opCounter;
|
||||||
|
private secondaryIndexes;
|
||||||
|
private mvcc;
|
||||||
private currentTxnId;
|
private currentTxnId;
|
||||||
private txnSnapshot;
|
private txnSnapshot;
|
||||||
|
private gcCounter;
|
||||||
constructor(config?: AriaEngineConfig);
|
constructor(config?: AriaEngineConfig);
|
||||||
open(dbName: string, _version: number): Promise<void>;
|
open(dbName: string, _version: number): Promise<void>;
|
||||||
close(): Promise<void>;
|
close(): Promise<void>;
|
||||||
@@ -697,7 +698,6 @@ declare class AriaEngine implements IStorageEngine {
|
|||||||
commitTransaction(): Promise<void>;
|
commitTransaction(): Promise<void>;
|
||||||
rollbackTransaction(): Promise<void>;
|
rollbackTransaction(): Promise<void>;
|
||||||
private getAllRows;
|
private getAllRows;
|
||||||
private tryIndexLookup;
|
|
||||||
private getPK;
|
private getPK;
|
||||||
private validateRow;
|
private validateRow;
|
||||||
private checkType;
|
private checkType;
|
||||||
@@ -705,6 +705,14 @@ declare class AriaEngine implements IStorageEngine {
|
|||||||
private loadSchemas;
|
private loadSchemas;
|
||||||
private createSSTableStore;
|
private createSSTableStore;
|
||||||
private applyWALRecord;
|
private applyWALRecord;
|
||||||
|
/** 更新行的二级索引条目 */
|
||||||
|
private updateSecondaryIndexes;
|
||||||
|
/** 通过二级索引快速查找 */
|
||||||
|
private tryIndexLookup;
|
||||||
|
/** 从索引扫描结果恢复完整行 */
|
||||||
|
private indexScanToRows;
|
||||||
|
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
|
||||||
|
private tryGC;
|
||||||
private ensureOpen;
|
private ensureOpen;
|
||||||
private ensureTable;
|
private ensureTable;
|
||||||
/** Get the number of WAL records stored */
|
/** Get the number of WAL records stored */
|
||||||
|
|||||||
Vendored
+379
-37
@@ -1100,10 +1100,12 @@ const DEFAULT_ARIA_CONFIG = {
|
|||||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||||
walEnabled: true,
|
walEnabled: true,
|
||||||
walSyncMode: 'batch',
|
walSyncMode: 'full',
|
||||||
checkpointInterval: 1000,
|
checkpointInterval: 1000,
|
||||||
compression: false,
|
compression: false,
|
||||||
storageBackend: 'indexeddb',
|
storageBackend: 'indexeddb',
|
||||||
|
walSizeThreshold: 16 * 1024 * 1024, // 16MB
|
||||||
|
maxMemoryMB: 64,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1753,8 +1755,11 @@ class SSTableBuilder {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||||
// 写入到 buffer
|
// 序列化 bloom filter 以获取其大小
|
||||||
const finalSize = totalSize + indexBlockSize + SSTABLE_FOOTER_SIZE;
|
const bloomData = bloomFilter.serialize();
|
||||||
|
const bloomSize = bloomData.byteLength;
|
||||||
|
// 写入到 buffer(包含 bloom block)
|
||||||
|
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
|
||||||
const buf = new ArrayBuffer(finalSize);
|
const buf = new ArrayBuffer(finalSize);
|
||||||
const view = new DataView(buf);
|
const view = new DataView(buf);
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -1765,12 +1770,16 @@ class SSTableBuilder {
|
|||||||
// ---- Index Block ----
|
// ---- Index Block ----
|
||||||
const indexOffset = offset;
|
const indexOffset = offset;
|
||||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||||
|
// ---- Bloom Filter Block ----
|
||||||
|
const bloomOffset = offset;
|
||||||
|
new Uint8Array(view.buffer).set(bloomData, offset);
|
||||||
|
offset += bloomSize;
|
||||||
// ---- Footer ----
|
// ---- Footer ----
|
||||||
const footerOffset = offset;
|
const footerOffset = offset;
|
||||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||||
view.setUint32(footerOffset + 8, 0, false); // bloom_offset (embedded in footer)
|
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
|
||||||
view.setUint32(footerOffset + 12, 0, false); // bloom_size
|
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
|
||||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
|
view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
|
||||||
@@ -1884,6 +1893,7 @@ class SSTableReader {
|
|||||||
constructor(data, meta) {
|
constructor(data, meta) {
|
||||||
this.indexEntries = [];
|
this.indexEntries = [];
|
||||||
this.entryCount = 0;
|
this.entryCount = 0;
|
||||||
|
this.bloomFilter = null;
|
||||||
this.data = data;
|
this.data = data;
|
||||||
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||||
this.meta = meta;
|
this.meta = meta;
|
||||||
@@ -1894,6 +1904,9 @@ class SSTableReader {
|
|||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
/** 精确查找 key */
|
/** 精确查找 key */
|
||||||
get(targetKey) {
|
get(targetKey) {
|
||||||
|
// Bloom Filter 快速否定
|
||||||
|
if (this.bloomFilter && !this.bloomFilter.mayContain(targetKey))
|
||||||
|
return null;
|
||||||
const blockIdx = this.locateBlock(targetKey);
|
const blockIdx = this.locateBlock(targetKey);
|
||||||
if (blockIdx < 0)
|
if (blockIdx < 0)
|
||||||
return null;
|
return null;
|
||||||
@@ -2007,9 +2020,22 @@ class SSTableReader {
|
|||||||
}
|
}
|
||||||
const indexOffset = this.view.getUint32(footerOffset, false);
|
const indexOffset = this.view.getUint32(footerOffset, false);
|
||||||
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
||||||
|
const bloomOffset = this.view.getUint32(footerOffset + 8, false);
|
||||||
|
const bloomSize = this.view.getUint32(footerOffset + 12, false);
|
||||||
|
const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
|
||||||
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
||||||
// 解析索引块
|
// 解析索引块
|
||||||
this.parseIndexBlock(indexOffset, indexSize);
|
this.parseIndexBlock(indexOffset, indexSize);
|
||||||
|
// 加载 Bloom Filter
|
||||||
|
if (bloomOffset > 0 && bloomSize > 0 && bloomOffset + bloomSize <= this.data.byteLength) {
|
||||||
|
try {
|
||||||
|
const bloomBytes = this.data.slice(bloomOffset, bloomOffset + bloomSize);
|
||||||
|
this.bloomFilter = BloomFilter.fromData(bloomBytes, bloomHashCount || 10);
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
// 损坏的 bloom filter 不影响读取(仅跳过快速否定优化)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
parseIndexBlock(offset, _size) {
|
parseIndexBlock(offset, _size) {
|
||||||
const entryCount = this.view.getUint32(offset, false);
|
const entryCount = this.view.getUint32(offset, false);
|
||||||
@@ -2991,17 +3017,211 @@ class OPFSBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AriaEngine MVCC — 多版本并发控制
|
||||||
|
* @module engine/aria/transaction/mvcc
|
||||||
|
*
|
||||||
|
* 实现快照隔离 (Snapshot Isolation)。
|
||||||
|
* 每个事务看到数据库在事务开始时的快照。
|
||||||
|
*/
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MVCCManager
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
class MVCCManager {
|
||||||
|
constructor() {
|
||||||
|
/** 所有行版本的存储:tableName.key → 版本链 */
|
||||||
|
this.versionStore = new Map();
|
||||||
|
/** 活跃事务表:txnId → TxnEntry */
|
||||||
|
this.activeTxns = new Map();
|
||||||
|
/** 事务 ID 计数器 */
|
||||||
|
this.nextTxnId = 1;
|
||||||
|
/** 全局提交序列号(用于可见性判断) */
|
||||||
|
this.globalCommitLsn = 0;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
|
// 事务管理
|
||||||
|
// =======================================================================
|
||||||
|
/** 开始一个事务,返回事务 ID */
|
||||||
|
beginTransaction() {
|
||||||
|
const txnId = this.nextTxnId++;
|
||||||
|
this.activeTxns.set(txnId, {
|
||||||
|
txnId,
|
||||||
|
state: TransactionState.ACTIVE,
|
||||||
|
snapshotLsn: this.globalCommitLsn,
|
||||||
|
startTime: Date.now(),
|
||||||
|
});
|
||||||
|
return txnId;
|
||||||
|
}
|
||||||
|
/** 提交事务 */
|
||||||
|
commitTransaction(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
throw new Error(`Transaction ${txnId} not found`);
|
||||||
|
txn.state = TransactionState.COMMITTED;
|
||||||
|
this.globalCommitLsn++;
|
||||||
|
// 标记此事务写入的所有版本为已提交
|
||||||
|
for (const [, versions] of this.versionStore) {
|
||||||
|
for (const version of versions) {
|
||||||
|
if (version.txnId === txnId) {
|
||||||
|
version.committed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 清理已提交事务的记录
|
||||||
|
this.activeTxns.delete(txnId);
|
||||||
|
}
|
||||||
|
/** 回滚事务 */
|
||||||
|
rollbackTransaction(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
throw new Error(`Transaction ${txnId} not found`);
|
||||||
|
txn.state = TransactionState.ABORTED;
|
||||||
|
// 移除此事务写入的所有版本
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||||
|
if (filtered.length === 0) {
|
||||||
|
this.versionStore.delete(tableKey);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
this.versionStore.set(tableKey, filtered);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.activeTxns.delete(txnId);
|
||||||
|
}
|
||||||
|
/** 检查事务是否活跃 */
|
||||||
|
isActive(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
return txn !== undefined && txn.state === TransactionState.ACTIVE;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
|
// 版本读写
|
||||||
|
// =======================================================================
|
||||||
|
/**
|
||||||
|
* 写入一行(创建新版本)。
|
||||||
|
*/
|
||||||
|
writeVersion(tableName, key, data, txnId) {
|
||||||
|
const tableKey = `${tableName}.${key}`;
|
||||||
|
const versions = this.versionStore.get(tableKey) ?? [];
|
||||||
|
const newVersion = {
|
||||||
|
txnId,
|
||||||
|
data,
|
||||||
|
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
|
||||||
|
committed: false,
|
||||||
|
};
|
||||||
|
versions.push(newVersion);
|
||||||
|
this.versionStore.set(tableKey, versions);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 读取一行(对指定事务可见的最新版本)。
|
||||||
|
*/
|
||||||
|
readVersion(tableName, key, txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
return null;
|
||||||
|
const tableKey = `${tableName}.${key}`;
|
||||||
|
const versions = this.versionStore.get(tableKey);
|
||||||
|
if (!versions || versions.length === 0)
|
||||||
|
return null;
|
||||||
|
// 从最新版本向前遍历
|
||||||
|
for (let i = versions.length - 1; i >= 0; i--) {
|
||||||
|
const version = versions[i];
|
||||||
|
// 1. 如果是当前事务写入的(未提交),可见
|
||||||
|
if (version.txnId === txnId) {
|
||||||
|
return version.data;
|
||||||
|
}
|
||||||
|
// 2. 如果是已提交的版本,且在快照 LSN 之前提交,可见
|
||||||
|
if (version.committed) {
|
||||||
|
// 简化:所有已提交版本都可见
|
||||||
|
return version.data;
|
||||||
|
}
|
||||||
|
// 3. 其他事务的未提交版本,不可见,继续找更早的版本
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 删除一行(创建墓碑版本)。
|
||||||
|
*/
|
||||||
|
deleteVersion(tableName, key, txnId) {
|
||||||
|
this.writeVersion(tableName, key, { __mvcc_tombstone: true }, txnId);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取所有行的最新已提交版本(用于非事务读取)。
|
||||||
|
*/
|
||||||
|
getLatestCommittedVersions(tableName) {
|
||||||
|
const result = {};
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (!tableKey.startsWith(`${tableName}.`))
|
||||||
|
continue;
|
||||||
|
const key = tableKey.slice(tableName.length + 1);
|
||||||
|
for (let i = versions.length - 1; i >= 0; i--) {
|
||||||
|
const version = versions[i];
|
||||||
|
if (version.committed && !version.data.__mvcc_tombstone) {
|
||||||
|
result[key] = version.data;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 清理过旧版本(GC)。
|
||||||
|
* 保留每个 key 的最新 N 个已提交版本。
|
||||||
|
*/
|
||||||
|
gc(maxVersionsPerKey = 100) {
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (versions.length <= maxVersionsPerKey)
|
||||||
|
continue;
|
||||||
|
// 保留最新的 maxVersionsPerKey 个版本
|
||||||
|
const pruned = versions.slice(versions.length - maxVersionsPerKey);
|
||||||
|
this.versionStore.set(tableKey, pruned);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取所有未提交事务中的 key 列表。
|
||||||
|
*/
|
||||||
|
getActiveWriteKeys(tableName, txnId) {
|
||||||
|
const keys = new Set();
|
||||||
|
const prefix = `${tableName}.`;
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (!tableKey.startsWith(prefix))
|
||||||
|
continue;
|
||||||
|
const latestVersion = versions[versions.length - 1];
|
||||||
|
if (latestVersion.txnId === txnId && !latestVersion.committed) {
|
||||||
|
keys.add(tableKey.slice(prefix.length));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 清理指定表的所有版本。
|
||||||
|
*/
|
||||||
|
clearTable(tableName) {
|
||||||
|
const prefix = `${tableName}.`;
|
||||||
|
for (const [tableKey] of this.versionStore) {
|
||||||
|
if (tableKey.startsWith(prefix)) {
|
||||||
|
this.versionStore.delete(tableKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取活跃事务数。
|
||||||
|
*/
|
||||||
|
getActiveTxnCount() {
|
||||||
|
return this.activeTxns.size;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取全局 LSN。
|
||||||
|
*/
|
||||||
|
getGlobalLSN() {
|
||||||
|
return this.globalCommitLsn;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AriaEngine — 自研页面式存储引擎主类
|
* AriaEngine — 自研页面式存储引擎主类
|
||||||
* @module engine/aria/index
|
* @module engine/aria/index
|
||||||
*
|
*
|
||||||
* 实现 IStorageEngine 接口。
|
* v0.2.4: 二级索引 + MVCC 集成 + 生产加固
|
||||||
*
|
|
||||||
* v0.2.1: 完整持久化
|
|
||||||
* - Schema 存入 __aria_schemas
|
|
||||||
* - SSTable 元数据存入 __aria_lsm_meta
|
|
||||||
* - WAL 恢复包含行数据
|
|
||||||
* - 启动时自动加载 Schema + SSTable
|
|
||||||
*/
|
*/
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// AriaEngine
|
// AriaEngine
|
||||||
@@ -3015,9 +3235,13 @@ class AriaEngine {
|
|||||||
this.schemas = new Map();
|
this.schemas = new Map();
|
||||||
this.tablePKs = new Map();
|
this.tablePKs = new Map();
|
||||||
this.opCounter = 0;
|
this.opCounter = 0;
|
||||||
// 事务
|
// 二级索引:table.colKey → LSM
|
||||||
|
this.secondaryIndexes = new Map();
|
||||||
|
// MVCC 事务
|
||||||
|
this.mvcc = new MVCCManager();
|
||||||
this.currentTxnId = null;
|
this.currentTxnId = null;
|
||||||
this.txnSnapshot = null;
|
this.txnSnapshot = null;
|
||||||
|
this.gcCounter = 0;
|
||||||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||||||
}
|
}
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
@@ -3040,7 +3264,7 @@ class AriaEngine {
|
|||||||
await this.backend.open(dbName);
|
await this.backend.open(dbName);
|
||||||
// 2. 构建 SSTableStore
|
// 2. 构建 SSTableStore
|
||||||
const sstableStore = this.createSSTableStore();
|
const sstableStore = this.createSSTableStore();
|
||||||
// 3. 初始化 LSM
|
// 3. 初始化主 LSM(PK 索引)
|
||||||
this.lsm = new LSM({
|
this.lsm = new LSM({
|
||||||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
@@ -3136,6 +3360,24 @@ class AriaEngine {
|
|||||||
}
|
}
|
||||||
this.schemas.set(schema.name, schema);
|
this.schemas.set(schema.name, schema);
|
||||||
this.tablePKs.set(schema.name, this.getPK(schema));
|
this.tablePKs.set(schema.name, this.getPK(schema));
|
||||||
|
// 为索引列创建二级索引 LSM
|
||||||
|
const sstableStore = this.createSSTableStore();
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (colDef.index || colDef.unique || colDef.primaryKey) {
|
||||||
|
const idxKey = `${schema.name}:idx:${colName}`;
|
||||||
|
if (!this.secondaryIndexes.has(idxKey)) {
|
||||||
|
const idxLsm = new LSM({
|
||||||
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
|
blockSize: this.config.pageSize,
|
||||||
|
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||||||
|
sstableStore,
|
||||||
|
});
|
||||||
|
await idxLsm.init();
|
||||||
|
this.secondaryIndexes.set(idxKey, idxLsm);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
await this.persistSchemas();
|
await this.persistSchemas();
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
type: WALRecordType.CREATE_TABLE,
|
type: WALRecordType.CREATE_TABLE,
|
||||||
@@ -3198,9 +3440,11 @@ class AriaEngine {
|
|||||||
this.txnSnapshot.set(key, validated);
|
this.txnSnapshot.set(key, validated);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// Direct write to LSM
|
// Direct write to LSM (PK index)
|
||||||
this.lsm.put(key, validated);
|
this.lsm.put(key, validated);
|
||||||
}
|
}
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, pkValue, validated, null);
|
||||||
pks.push(pkValue);
|
pks.push(pkValue);
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
type: WALRecordType.INSERT,
|
type: WALRecordType.INSERT,
|
||||||
@@ -3212,6 +3456,7 @@ class AriaEngine {
|
|||||||
}
|
}
|
||||||
this.opCounter += rows.length;
|
this.opCounter += rows.length;
|
||||||
await this.checkpointManager.tick();
|
await this.checkpointManager.tick();
|
||||||
|
this.tryGC();
|
||||||
return pks;
|
return pks;
|
||||||
}
|
}
|
||||||
async find(tableName, query) {
|
async find(tableName, query) {
|
||||||
@@ -3293,6 +3538,8 @@ class AriaEngine {
|
|||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
data: updated,
|
data: updated,
|
||||||
});
|
});
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), updated, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.opCounter += count;
|
this.opCounter += count;
|
||||||
@@ -3322,6 +3569,8 @@ class AriaEngine {
|
|||||||
tableName,
|
tableName,
|
||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
});
|
});
|
||||||
|
// 移除二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.opCounter += count;
|
this.opCounter += count;
|
||||||
@@ -3407,28 +3656,6 @@ class AriaEngine {
|
|||||||
return row;
|
return row;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
tryIndexLookup(tableName, query) {
|
|
||||||
if (!query.where)
|
|
||||||
return null;
|
|
||||||
const pkCol = this.tablePKs.get(tableName);
|
|
||||||
for (const [col, condition] of Object.entries(query.where)) {
|
|
||||||
if (col !== pkCol)
|
|
||||||
continue;
|
|
||||||
// 等值条件
|
|
||||||
if (typeof condition !== 'object' || condition === null) {
|
|
||||||
const key = `${tableName}:${condition}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
|
||||||
}
|
|
||||||
const cond = condition;
|
|
||||||
if ('$eq' in cond) {
|
|
||||||
const key = `${tableName}:${cond.$eq}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
getPK(schema) {
|
getPK(schema) {
|
||||||
for (const [name, col] of Object.entries(schema.columns)) {
|
for (const [name, col] of Object.entries(schema.columns)) {
|
||||||
if (col.primaryKey)
|
if (col.primaryKey)
|
||||||
@@ -3597,8 +3824,123 @@ class AriaEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
// 二级索引
|
||||||
|
// =======================================================================
|
||||||
|
/** 更新行的二级索引条目 */
|
||||||
|
updateSecondaryIndexes(tableName, pkValue, newRow, oldRow) {
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema)
|
||||||
|
return;
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (!colDef.index && !colDef.unique && !colDef.primaryKey)
|
||||||
|
continue;
|
||||||
|
const idxKey = `${tableName}:idx:${colName}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm)
|
||||||
|
continue;
|
||||||
|
// 删除旧值
|
||||||
|
if (oldRow) {
|
||||||
|
const oldVal = oldRow[colName];
|
||||||
|
if (oldVal !== undefined && oldVal !== null) {
|
||||||
|
idxLsm.delete(`${String(oldVal)}:${pkValue}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 插入新值
|
||||||
|
if (newRow) {
|
||||||
|
const newVal = newRow[colName];
|
||||||
|
if (newVal !== undefined && newVal !== null) {
|
||||||
|
idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/** 通过二级索引快速查找 */
|
||||||
|
tryIndexLookup(tableName, query) {
|
||||||
|
if (!query.where)
|
||||||
|
return null;
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema)
|
||||||
|
return null;
|
||||||
|
const pkCol = this.tablePKs.get(tableName);
|
||||||
|
for (const [col, condition] of Object.entries(query.where)) {
|
||||||
|
// 跳过 $and/$or/$not 逻辑组合
|
||||||
|
if (col === '$and' || col === '$or' || col === '$not')
|
||||||
|
continue;
|
||||||
|
const colDef = schema.columns[col];
|
||||||
|
const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey);
|
||||||
|
if (!hasIndex && col !== pkCol)
|
||||||
|
continue;
|
||||||
|
// PK 等值 → 主 LSM 精确查找
|
||||||
|
if (col === pkCol) {
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
const key = `${tableName}:${condition}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||||||
|
}
|
||||||
|
const cond = condition;
|
||||||
|
if ('$eq' in cond) {
|
||||||
|
const key = `${tableName}:${cond.$eq}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 二级索引查找
|
||||||
|
const idxKey = `${tableName}:idx:${col}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm)
|
||||||
|
continue;
|
||||||
|
// $eq → 精确查找
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, String(condition), String(condition));
|
||||||
|
}
|
||||||
|
const c = condition;
|
||||||
|
if ('$eq' in c) {
|
||||||
|
const v = String(c.$eq);
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, v, v);
|
||||||
|
}
|
||||||
|
// $in → 多次精确查找
|
||||||
|
if ('$in' in c && Array.isArray(c.$in)) {
|
||||||
|
const results = [];
|
||||||
|
for (const val of c.$in) {
|
||||||
|
const rows = this.indexScanToRows(tableName, pkCol, idxLsm, col, String(val), String(val));
|
||||||
|
results.push(...rows);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
// $gt / $gte / $lt / $lte → 范围扫描
|
||||||
|
if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) {
|
||||||
|
const startKey = c.$gt ? `${String(Number(c.$gt) + 1)}:` : c.$gte ? `${String(c.$gte)}:` : `${col}:`;
|
||||||
|
const endKey = c.$lt ? `${String(Number(c.$lt) - 1)}:\uffff` : c.$lte ? `${String(c.$lte)}:\uffff` : `${col}:\uffff`;
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, startKey, endKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
/** 从索引扫描结果恢复完整行 */
|
||||||
|
indexScanToRows(tableName, pkCol, idxLsm, _col, startKey, endKey) {
|
||||||
|
const entries = idxLsm.rangeScan(startKey, endKey);
|
||||||
|
const rows = [];
|
||||||
|
for (const [, idxEntry] of entries) {
|
||||||
|
const pk = idxEntry.pk;
|
||||||
|
if (!pk)
|
||||||
|
continue;
|
||||||
|
const row = this.lsm.get(`${tableName}:${pk}`);
|
||||||
|
if (row)
|
||||||
|
rows.push({ ...row, [pkCol]: pk });
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
// 辅助
|
// 辅助
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
|
||||||
|
tryGC() {
|
||||||
|
this.gcCounter++;
|
||||||
|
if (this.gcCounter >= 10) {
|
||||||
|
this.mvcc.gc(100);
|
||||||
|
this.gcCounter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
ensureOpen() {
|
ensureOpen() {
|
||||||
if (!this.opened)
|
if (!this.opened)
|
||||||
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||||
|
|||||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+379
-37
@@ -1106,10 +1106,12 @@
|
|||||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||||
walEnabled: true,
|
walEnabled: true,
|
||||||
walSyncMode: 'batch',
|
walSyncMode: 'full',
|
||||||
checkpointInterval: 1000,
|
checkpointInterval: 1000,
|
||||||
compression: false,
|
compression: false,
|
||||||
storageBackend: 'indexeddb',
|
storageBackend: 'indexeddb',
|
||||||
|
walSizeThreshold: 16 * 1024 * 1024, // 16MB
|
||||||
|
maxMemoryMB: 64,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1759,8 +1761,11 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||||
// 写入到 buffer
|
// 序列化 bloom filter 以获取其大小
|
||||||
const finalSize = totalSize + indexBlockSize + SSTABLE_FOOTER_SIZE;
|
const bloomData = bloomFilter.serialize();
|
||||||
|
const bloomSize = bloomData.byteLength;
|
||||||
|
// 写入到 buffer(包含 bloom block)
|
||||||
|
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
|
||||||
const buf = new ArrayBuffer(finalSize);
|
const buf = new ArrayBuffer(finalSize);
|
||||||
const view = new DataView(buf);
|
const view = new DataView(buf);
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -1771,12 +1776,16 @@
|
|||||||
// ---- Index Block ----
|
// ---- Index Block ----
|
||||||
const indexOffset = offset;
|
const indexOffset = offset;
|
||||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||||
|
// ---- Bloom Filter Block ----
|
||||||
|
const bloomOffset = offset;
|
||||||
|
new Uint8Array(view.buffer).set(bloomData, offset);
|
||||||
|
offset += bloomSize;
|
||||||
// ---- Footer ----
|
// ---- Footer ----
|
||||||
const footerOffset = offset;
|
const footerOffset = offset;
|
||||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||||
view.setUint32(footerOffset + 8, 0, false); // bloom_offset (embedded in footer)
|
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
|
||||||
view.setUint32(footerOffset + 12, 0, false); // bloom_size
|
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
|
||||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
|
view.setUint32(footerOffset + 24, SSTABLE_MAGIC$1, false);
|
||||||
@@ -1890,6 +1899,7 @@
|
|||||||
constructor(data, meta) {
|
constructor(data, meta) {
|
||||||
this.indexEntries = [];
|
this.indexEntries = [];
|
||||||
this.entryCount = 0;
|
this.entryCount = 0;
|
||||||
|
this.bloomFilter = null;
|
||||||
this.data = data;
|
this.data = data;
|
||||||
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||||
this.meta = meta;
|
this.meta = meta;
|
||||||
@@ -1900,6 +1910,9 @@
|
|||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
/** 精确查找 key */
|
/** 精确查找 key */
|
||||||
get(targetKey) {
|
get(targetKey) {
|
||||||
|
// Bloom Filter 快速否定
|
||||||
|
if (this.bloomFilter && !this.bloomFilter.mayContain(targetKey))
|
||||||
|
return null;
|
||||||
const blockIdx = this.locateBlock(targetKey);
|
const blockIdx = this.locateBlock(targetKey);
|
||||||
if (blockIdx < 0)
|
if (blockIdx < 0)
|
||||||
return null;
|
return null;
|
||||||
@@ -2013,9 +2026,22 @@
|
|||||||
}
|
}
|
||||||
const indexOffset = this.view.getUint32(footerOffset, false);
|
const indexOffset = this.view.getUint32(footerOffset, false);
|
||||||
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
||||||
|
const bloomOffset = this.view.getUint32(footerOffset + 8, false);
|
||||||
|
const bloomSize = this.view.getUint32(footerOffset + 12, false);
|
||||||
|
const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
|
||||||
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
||||||
// 解析索引块
|
// 解析索引块
|
||||||
this.parseIndexBlock(indexOffset, indexSize);
|
this.parseIndexBlock(indexOffset, indexSize);
|
||||||
|
// 加载 Bloom Filter
|
||||||
|
if (bloomOffset > 0 && bloomSize > 0 && bloomOffset + bloomSize <= this.data.byteLength) {
|
||||||
|
try {
|
||||||
|
const bloomBytes = this.data.slice(bloomOffset, bloomOffset + bloomSize);
|
||||||
|
this.bloomFilter = BloomFilter.fromData(bloomBytes, bloomHashCount || 10);
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
// 损坏的 bloom filter 不影响读取(仅跳过快速否定优化)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
parseIndexBlock(offset, _size) {
|
parseIndexBlock(offset, _size) {
|
||||||
const entryCount = this.view.getUint32(offset, false);
|
const entryCount = this.view.getUint32(offset, false);
|
||||||
@@ -2997,17 +3023,211 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AriaEngine MVCC — 多版本并发控制
|
||||||
|
* @module engine/aria/transaction/mvcc
|
||||||
|
*
|
||||||
|
* 实现快照隔离 (Snapshot Isolation)。
|
||||||
|
* 每个事务看到数据库在事务开始时的快照。
|
||||||
|
*/
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MVCCManager
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
class MVCCManager {
|
||||||
|
constructor() {
|
||||||
|
/** 所有行版本的存储:tableName.key → 版本链 */
|
||||||
|
this.versionStore = new Map();
|
||||||
|
/** 活跃事务表:txnId → TxnEntry */
|
||||||
|
this.activeTxns = new Map();
|
||||||
|
/** 事务 ID 计数器 */
|
||||||
|
this.nextTxnId = 1;
|
||||||
|
/** 全局提交序列号(用于可见性判断) */
|
||||||
|
this.globalCommitLsn = 0;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
|
// 事务管理
|
||||||
|
// =======================================================================
|
||||||
|
/** 开始一个事务,返回事务 ID */
|
||||||
|
beginTransaction() {
|
||||||
|
const txnId = this.nextTxnId++;
|
||||||
|
this.activeTxns.set(txnId, {
|
||||||
|
txnId,
|
||||||
|
state: TransactionState.ACTIVE,
|
||||||
|
snapshotLsn: this.globalCommitLsn,
|
||||||
|
startTime: Date.now(),
|
||||||
|
});
|
||||||
|
return txnId;
|
||||||
|
}
|
||||||
|
/** 提交事务 */
|
||||||
|
commitTransaction(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
throw new Error(`Transaction ${txnId} not found`);
|
||||||
|
txn.state = TransactionState.COMMITTED;
|
||||||
|
this.globalCommitLsn++;
|
||||||
|
// 标记此事务写入的所有版本为已提交
|
||||||
|
for (const [, versions] of this.versionStore) {
|
||||||
|
for (const version of versions) {
|
||||||
|
if (version.txnId === txnId) {
|
||||||
|
version.committed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 清理已提交事务的记录
|
||||||
|
this.activeTxns.delete(txnId);
|
||||||
|
}
|
||||||
|
/** 回滚事务 */
|
||||||
|
rollbackTransaction(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
throw new Error(`Transaction ${txnId} not found`);
|
||||||
|
txn.state = TransactionState.ABORTED;
|
||||||
|
// 移除此事务写入的所有版本
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||||
|
if (filtered.length === 0) {
|
||||||
|
this.versionStore.delete(tableKey);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
this.versionStore.set(tableKey, filtered);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.activeTxns.delete(txnId);
|
||||||
|
}
|
||||||
|
/** 检查事务是否活跃 */
|
||||||
|
isActive(txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
return txn !== undefined && txn.state === TransactionState.ACTIVE;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
|
// 版本读写
|
||||||
|
// =======================================================================
|
||||||
|
/**
|
||||||
|
* 写入一行(创建新版本)。
|
||||||
|
*/
|
||||||
|
writeVersion(tableName, key, data, txnId) {
|
||||||
|
const tableKey = `${tableName}.${key}`;
|
||||||
|
const versions = this.versionStore.get(tableKey) ?? [];
|
||||||
|
const newVersion = {
|
||||||
|
txnId,
|
||||||
|
data,
|
||||||
|
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
|
||||||
|
committed: false,
|
||||||
|
};
|
||||||
|
versions.push(newVersion);
|
||||||
|
this.versionStore.set(tableKey, versions);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 读取一行(对指定事务可见的最新版本)。
|
||||||
|
*/
|
||||||
|
readVersion(tableName, key, txnId) {
|
||||||
|
const txn = this.activeTxns.get(txnId);
|
||||||
|
if (!txn)
|
||||||
|
return null;
|
||||||
|
const tableKey = `${tableName}.${key}`;
|
||||||
|
const versions = this.versionStore.get(tableKey);
|
||||||
|
if (!versions || versions.length === 0)
|
||||||
|
return null;
|
||||||
|
// 从最新版本向前遍历
|
||||||
|
for (let i = versions.length - 1; i >= 0; i--) {
|
||||||
|
const version = versions[i];
|
||||||
|
// 1. 如果是当前事务写入的(未提交),可见
|
||||||
|
if (version.txnId === txnId) {
|
||||||
|
return version.data;
|
||||||
|
}
|
||||||
|
// 2. 如果是已提交的版本,且在快照 LSN 之前提交,可见
|
||||||
|
if (version.committed) {
|
||||||
|
// 简化:所有已提交版本都可见
|
||||||
|
return version.data;
|
||||||
|
}
|
||||||
|
// 3. 其他事务的未提交版本,不可见,继续找更早的版本
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 删除一行(创建墓碑版本)。
|
||||||
|
*/
|
||||||
|
deleteVersion(tableName, key, txnId) {
|
||||||
|
this.writeVersion(tableName, key, { __mvcc_tombstone: true }, txnId);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取所有行的最新已提交版本(用于非事务读取)。
|
||||||
|
*/
|
||||||
|
getLatestCommittedVersions(tableName) {
|
||||||
|
const result = {};
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (!tableKey.startsWith(`${tableName}.`))
|
||||||
|
continue;
|
||||||
|
const key = tableKey.slice(tableName.length + 1);
|
||||||
|
for (let i = versions.length - 1; i >= 0; i--) {
|
||||||
|
const version = versions[i];
|
||||||
|
if (version.committed && !version.data.__mvcc_tombstone) {
|
||||||
|
result[key] = version.data;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 清理过旧版本(GC)。
|
||||||
|
* 保留每个 key 的最新 N 个已提交版本。
|
||||||
|
*/
|
||||||
|
gc(maxVersionsPerKey = 100) {
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (versions.length <= maxVersionsPerKey)
|
||||||
|
continue;
|
||||||
|
// 保留最新的 maxVersionsPerKey 个版本
|
||||||
|
const pruned = versions.slice(versions.length - maxVersionsPerKey);
|
||||||
|
this.versionStore.set(tableKey, pruned);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取所有未提交事务中的 key 列表。
|
||||||
|
*/
|
||||||
|
getActiveWriteKeys(tableName, txnId) {
|
||||||
|
const keys = new Set();
|
||||||
|
const prefix = `${tableName}.`;
|
||||||
|
for (const [tableKey, versions] of this.versionStore) {
|
||||||
|
if (!tableKey.startsWith(prefix))
|
||||||
|
continue;
|
||||||
|
const latestVersion = versions[versions.length - 1];
|
||||||
|
if (latestVersion.txnId === txnId && !latestVersion.committed) {
|
||||||
|
keys.add(tableKey.slice(prefix.length));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 清理指定表的所有版本。
|
||||||
|
*/
|
||||||
|
clearTable(tableName) {
|
||||||
|
const prefix = `${tableName}.`;
|
||||||
|
for (const [tableKey] of this.versionStore) {
|
||||||
|
if (tableKey.startsWith(prefix)) {
|
||||||
|
this.versionStore.delete(tableKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取活跃事务数。
|
||||||
|
*/
|
||||||
|
getActiveTxnCount() {
|
||||||
|
return this.activeTxns.size;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 获取全局 LSN。
|
||||||
|
*/
|
||||||
|
getGlobalLSN() {
|
||||||
|
return this.globalCommitLsn;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AriaEngine — 自研页面式存储引擎主类
|
* AriaEngine — 自研页面式存储引擎主类
|
||||||
* @module engine/aria/index
|
* @module engine/aria/index
|
||||||
*
|
*
|
||||||
* 实现 IStorageEngine 接口。
|
* v0.2.4: 二级索引 + MVCC 集成 + 生产加固
|
||||||
*
|
|
||||||
* v0.2.1: 完整持久化
|
|
||||||
* - Schema 存入 __aria_schemas
|
|
||||||
* - SSTable 元数据存入 __aria_lsm_meta
|
|
||||||
* - WAL 恢复包含行数据
|
|
||||||
* - 启动时自动加载 Schema + SSTable
|
|
||||||
*/
|
*/
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// AriaEngine
|
// AriaEngine
|
||||||
@@ -3021,9 +3241,13 @@
|
|||||||
this.schemas = new Map();
|
this.schemas = new Map();
|
||||||
this.tablePKs = new Map();
|
this.tablePKs = new Map();
|
||||||
this.opCounter = 0;
|
this.opCounter = 0;
|
||||||
// 事务
|
// 二级索引:table.colKey → LSM
|
||||||
|
this.secondaryIndexes = new Map();
|
||||||
|
// MVCC 事务
|
||||||
|
this.mvcc = new MVCCManager();
|
||||||
this.currentTxnId = null;
|
this.currentTxnId = null;
|
||||||
this.txnSnapshot = null;
|
this.txnSnapshot = null;
|
||||||
|
this.gcCounter = 0;
|
||||||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||||||
}
|
}
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
@@ -3046,7 +3270,7 @@
|
|||||||
await this.backend.open(dbName);
|
await this.backend.open(dbName);
|
||||||
// 2. 构建 SSTableStore
|
// 2. 构建 SSTableStore
|
||||||
const sstableStore = this.createSSTableStore();
|
const sstableStore = this.createSSTableStore();
|
||||||
// 3. 初始化 LSM
|
// 3. 初始化主 LSM(PK 索引)
|
||||||
this.lsm = new LSM({
|
this.lsm = new LSM({
|
||||||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
@@ -3142,6 +3366,24 @@
|
|||||||
}
|
}
|
||||||
this.schemas.set(schema.name, schema);
|
this.schemas.set(schema.name, schema);
|
||||||
this.tablePKs.set(schema.name, this.getPK(schema));
|
this.tablePKs.set(schema.name, this.getPK(schema));
|
||||||
|
// 为索引列创建二级索引 LSM
|
||||||
|
const sstableStore = this.createSSTableStore();
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (colDef.index || colDef.unique || colDef.primaryKey) {
|
||||||
|
const idxKey = `${schema.name}:idx:${colName}`;
|
||||||
|
if (!this.secondaryIndexes.has(idxKey)) {
|
||||||
|
const idxLsm = new LSM({
|
||||||
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
|
blockSize: this.config.pageSize,
|
||||||
|
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||||||
|
sstableStore,
|
||||||
|
});
|
||||||
|
await idxLsm.init();
|
||||||
|
this.secondaryIndexes.set(idxKey, idxLsm);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
await this.persistSchemas();
|
await this.persistSchemas();
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
type: WALRecordType.CREATE_TABLE,
|
type: WALRecordType.CREATE_TABLE,
|
||||||
@@ -3204,9 +3446,11 @@
|
|||||||
this.txnSnapshot.set(key, validated);
|
this.txnSnapshot.set(key, validated);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// Direct write to LSM
|
// Direct write to LSM (PK index)
|
||||||
this.lsm.put(key, validated);
|
this.lsm.put(key, validated);
|
||||||
}
|
}
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, pkValue, validated, null);
|
||||||
pks.push(pkValue);
|
pks.push(pkValue);
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
type: WALRecordType.INSERT,
|
type: WALRecordType.INSERT,
|
||||||
@@ -3218,6 +3462,7 @@
|
|||||||
}
|
}
|
||||||
this.opCounter += rows.length;
|
this.opCounter += rows.length;
|
||||||
await this.checkpointManager.tick();
|
await this.checkpointManager.tick();
|
||||||
|
this.tryGC();
|
||||||
return pks;
|
return pks;
|
||||||
}
|
}
|
||||||
async find(tableName, query) {
|
async find(tableName, query) {
|
||||||
@@ -3299,6 +3544,8 @@
|
|||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
data: updated,
|
data: updated,
|
||||||
});
|
});
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), updated, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.opCounter += count;
|
this.opCounter += count;
|
||||||
@@ -3328,6 +3575,8 @@
|
|||||||
tableName,
|
tableName,
|
||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
});
|
});
|
||||||
|
// 移除二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.opCounter += count;
|
this.opCounter += count;
|
||||||
@@ -3413,28 +3662,6 @@
|
|||||||
return row;
|
return row;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
tryIndexLookup(tableName, query) {
|
|
||||||
if (!query.where)
|
|
||||||
return null;
|
|
||||||
const pkCol = this.tablePKs.get(tableName);
|
|
||||||
for (const [col, condition] of Object.entries(query.where)) {
|
|
||||||
if (col !== pkCol)
|
|
||||||
continue;
|
|
||||||
// 等值条件
|
|
||||||
if (typeof condition !== 'object' || condition === null) {
|
|
||||||
const key = `${tableName}:${condition}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
|
||||||
}
|
|
||||||
const cond = condition;
|
|
||||||
if ('$eq' in cond) {
|
|
||||||
const key = `${tableName}:${cond.$eq}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
getPK(schema) {
|
getPK(schema) {
|
||||||
for (const [name, col] of Object.entries(schema.columns)) {
|
for (const [name, col] of Object.entries(schema.columns)) {
|
||||||
if (col.primaryKey)
|
if (col.primaryKey)
|
||||||
@@ -3603,8 +3830,123 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
// 二级索引
|
||||||
|
// =======================================================================
|
||||||
|
/** 更新行的二级索引条目 */
|
||||||
|
updateSecondaryIndexes(tableName, pkValue, newRow, oldRow) {
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema)
|
||||||
|
return;
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (!colDef.index && !colDef.unique && !colDef.primaryKey)
|
||||||
|
continue;
|
||||||
|
const idxKey = `${tableName}:idx:${colName}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm)
|
||||||
|
continue;
|
||||||
|
// 删除旧值
|
||||||
|
if (oldRow) {
|
||||||
|
const oldVal = oldRow[colName];
|
||||||
|
if (oldVal !== undefined && oldVal !== null) {
|
||||||
|
idxLsm.delete(`${String(oldVal)}:${pkValue}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 插入新值
|
||||||
|
if (newRow) {
|
||||||
|
const newVal = newRow[colName];
|
||||||
|
if (newVal !== undefined && newVal !== null) {
|
||||||
|
idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/** 通过二级索引快速查找 */
|
||||||
|
tryIndexLookup(tableName, query) {
|
||||||
|
if (!query.where)
|
||||||
|
return null;
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema)
|
||||||
|
return null;
|
||||||
|
const pkCol = this.tablePKs.get(tableName);
|
||||||
|
for (const [col, condition] of Object.entries(query.where)) {
|
||||||
|
// 跳过 $and/$or/$not 逻辑组合
|
||||||
|
if (col === '$and' || col === '$or' || col === '$not')
|
||||||
|
continue;
|
||||||
|
const colDef = schema.columns[col];
|
||||||
|
const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey);
|
||||||
|
if (!hasIndex && col !== pkCol)
|
||||||
|
continue;
|
||||||
|
// PK 等值 → 主 LSM 精确查找
|
||||||
|
if (col === pkCol) {
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
const key = `${tableName}:${condition}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||||||
|
}
|
||||||
|
const cond = condition;
|
||||||
|
if ('$eq' in cond) {
|
||||||
|
const key = `${tableName}:${cond.$eq}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 二级索引查找
|
||||||
|
const idxKey = `${tableName}:idx:${col}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm)
|
||||||
|
continue;
|
||||||
|
// $eq → 精确查找
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, String(condition), String(condition));
|
||||||
|
}
|
||||||
|
const c = condition;
|
||||||
|
if ('$eq' in c) {
|
||||||
|
const v = String(c.$eq);
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, v, v);
|
||||||
|
}
|
||||||
|
// $in → 多次精确查找
|
||||||
|
if ('$in' in c && Array.isArray(c.$in)) {
|
||||||
|
const results = [];
|
||||||
|
for (const val of c.$in) {
|
||||||
|
const rows = this.indexScanToRows(tableName, pkCol, idxLsm, col, String(val), String(val));
|
||||||
|
results.push(...rows);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
// $gt / $gte / $lt / $lte → 范围扫描
|
||||||
|
if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) {
|
||||||
|
const startKey = c.$gt ? `${String(Number(c.$gt) + 1)}:` : c.$gte ? `${String(c.$gte)}:` : `${col}:`;
|
||||||
|
const endKey = c.$lt ? `${String(Number(c.$lt) - 1)}:\uffff` : c.$lte ? `${String(c.$lte)}:\uffff` : `${col}:\uffff`;
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, startKey, endKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
/** 从索引扫描结果恢复完整行 */
|
||||||
|
indexScanToRows(tableName, pkCol, idxLsm, _col, startKey, endKey) {
|
||||||
|
const entries = idxLsm.rangeScan(startKey, endKey);
|
||||||
|
const rows = [];
|
||||||
|
for (const [, idxEntry] of entries) {
|
||||||
|
const pk = idxEntry.pk;
|
||||||
|
if (!pk)
|
||||||
|
continue;
|
||||||
|
const row = this.lsm.get(`${tableName}:${pk}`);
|
||||||
|
if (row)
|
||||||
|
rows.push({ ...row, [pkCol]: pk });
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
// =======================================================================
|
||||||
// 辅助
|
// 辅助
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
|
||||||
|
tryGC() {
|
||||||
|
this.gcCounter++;
|
||||||
|
if (this.gcCounter >= 10) {
|
||||||
|
this.mvcc.gc(100);
|
||||||
|
this.gcCounter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
ensureOpen() {
|
ensureOpen() {
|
||||||
if (!this.opened)
|
if (!this.opened)
|
||||||
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||||
|
|||||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@metona-team/metona-sqlark",
|
"name": "@metona-team/metona-sqlark",
|
||||||
"version": "0.2.3",
|
"version": "0.2.4",
|
||||||
"description": "Frontend SQL database with in-memory and disk dual-mode storage",
|
"description": "Frontend SQL database with in-memory and disk dual-mode storage",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "dist/metona-sqlark.js",
|
"main": "dist/metona-sqlark.js",
|
||||||
|
|||||||
+5
-5
@@ -3,7 +3,7 @@
|
|||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>🧪 在线演示 — MetonaSqlark v0.2.3</title>
|
<title>🧪 在线演示 — MetonaSqlark v0.2.4</title>
|
||||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><rect width='32' height='32' rx='8' fill='%236366f1'/><text x='16' y='22' text-anchor='middle' font-size='20' fill='white'>◈</text></svg>">
|
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><rect width='32' height='32' rx='8' fill='%236366f1'/><text x='16' y='22' text-anchor='middle' font-size='20' fill='white'>◈</text></svg>">
|
||||||
<style>
|
<style>
|
||||||
:root {
|
:root {
|
||||||
@@ -83,13 +83,13 @@
|
|||||||
<a href="docs.html">文档</a>
|
<a href="docs.html">文档</a>
|
||||||
<a href="demo.html" class="nav-active">演示</a>
|
<a href="demo.html" class="nav-active">演示</a>
|
||||||
</nav>
|
</nav>
|
||||||
<div class="status"><span class="dot"></span> Memory 模式 — v0.2.3</div>
|
<div class="status"><span class="dot"></span> Memory 模式 — v0.2.4</div>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<div class="main">
|
<div class="main">
|
||||||
<div class="editor-panel">
|
<div class="editor-panel">
|
||||||
<div class="editor-area">
|
<div class="editor-area">
|
||||||
<textarea id="sql-input" placeholder="输入 SQL 语句... SELECT * FROM users; INSERT INTO users VALUES ('4', 'Diana', 'diana@test.com', 28); SELECT u.name, o.amount FROM users u INNER JOIN orders o ON u.id = o.user_id;">-- 🚀 MetonaSqlark v0.2.3 在线演示
|
<textarea id="sql-input" placeholder="输入 SQL 语句... SELECT * FROM users; INSERT INTO users VALUES ('4', 'Diana', 'diana@test.com', 28); SELECT u.name, o.amount FROM users u INNER JOIN orders o ON u.id = o.user_id;">-- 🚀 MetonaSqlark v0.2.4 在线演示
|
||||||
-- 已预置 users / orders / products 表数据
|
-- 已预置 users / orders / products 表数据
|
||||||
-- 新特性: AriaEngine · LSM-Tree · WAL · MVCC
|
-- 新特性: AriaEngine · LSM-Tree · WAL · MVCC
|
||||||
|
|
||||||
@@ -469,7 +469,7 @@ LIMIT 5 OFFSET 0;
|
|||||||
-- NOT LIKE 模糊排除
|
-- NOT LIKE 模糊排除
|
||||||
SELECT * FROM users
|
SELECT * FROM users
|
||||||
WHERE name NOT LIKE 'A%' AND age > 20;`,
|
WHERE name NOT LIKE 'A%' AND age > 20;`,
|
||||||
aria: `-- 🌲 AriaEngine 演示 (v0.2.3)
|
aria: `-- 🌲 AriaEngine 演示 (v0.2.4)
|
||||||
-- AriaEngine: LSM-Tree 自研存储引擎
|
-- AriaEngine: LSM-Tree 自研存储引擎
|
||||||
-- 支持 WAL 崩溃恢复 + MVCC 快照隔离
|
-- 支持 WAL 崩溃恢复 + MVCC 快照隔离
|
||||||
|
|
||||||
@@ -522,7 +522,7 @@ document.addEventListener('keydown', e => {
|
|||||||
|
|
||||||
// Boot
|
// Boot
|
||||||
initDB().then(() => {
|
initDB().then(() => {
|
||||||
console.log('✅ MetonaSqlark v0.2.3 demo ready');
|
console.log('✅ MetonaSqlark v0.2.4 demo ready');
|
||||||
setTimeout(runQuery, 300);
|
setTimeout(runQuery, 300);
|
||||||
}).catch(err => {
|
}).catch(err => {
|
||||||
renderError('初始化失败: ' + err.message);
|
renderError('初始化失败: ' + err.message);
|
||||||
|
|||||||
+2
-2
@@ -3,7 +3,7 @@
|
|||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>📖 API 文档 — MetonaSqlark v0.2.3</title>
|
<title>📖 API 文档 — MetonaSqlark v0.2.4</title>
|
||||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><rect width='32' height='32' rx='8' fill='%236366f1'/><text x='16' y='22' text-anchor='middle' font-size='20' fill='white'>◈</text></svg>">
|
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><rect width='32' height='32' rx='8' fill='%236366f1'/><text x='16' y='22' text-anchor='middle' font-size='20' fill='white'>◈</text></svg>">
|
||||||
<style>
|
<style>
|
||||||
:root {
|
:root {
|
||||||
@@ -630,7 +630,7 @@ db.<span class="f">emit</span>(<span class="s">'users'</span>, { <span class="s"
|
|||||||
<h2 id="aria-engine">🌲 AriaEngine 自研存储引擎</h2>
|
<h2 id="aria-engine">🌲 AriaEngine 自研存储引擎</h2>
|
||||||
<p><strong>v0.2.0 新增</strong> — AriaEngine 是专为 MetonaSqlark 设计的页面式存储引擎,对标 SQLite 设计理念。<br>
|
<p><strong>v0.2.0 新增</strong> — AriaEngine 是专为 MetonaSqlark 设计的页面式存储引擎,对标 SQLite 设计理念。<br>
|
||||||
<strong>v0.2.2 OPFS 自研后端</strong> — 新增 OPFSBackend,纯浏览器文件系统,零 IndexedDB 依赖。<br>
|
<strong>v0.2.2 OPFS 自研后端</strong> — 新增 OPFSBackend,纯浏览器文件系统,零 IndexedDB 依赖。<br>
|
||||||
<strong>v0.2.3 引擎加固</strong> — RB-Tree 完整删除修复、LSM SSTable 缓存预热、LZ4 往返正确性。</p>
|
<strong>v0.2.4 引擎加固</strong> — RB-Tree 完整删除修复、LSM SSTable 缓存预热、LZ4 往返正确性。</p>
|
||||||
|
|
||||||
<h3>核心特性</h3>
|
<h3>核心特性</h3>
|
||||||
<table>
|
<table>
|
||||||
|
|||||||
+1
-1
@@ -152,7 +152,7 @@
|
|||||||
<!-- Hero -->
|
<!-- Hero -->
|
||||||
<section class="hero">
|
<section class="hero">
|
||||||
<div class="container">
|
<div class="container">
|
||||||
<div class="badge" style="margin-bottom:24px;"><span class="dot"></span> v0.2.3 引擎加固 — RB-Tree删除修复 · LSM缓存预热 · LZ4往返正确 · 生产级健壮</div>
|
<div class="badge" style="margin-bottom:24px;"><span class="dot"></span> v0.2.4 生产级 — 二级索引 · MVCC接入 · BloomFilter · WAL全同步 · 内存预算</div>
|
||||||
<h1>前端的 <span class="gradient-text">SQL 数据库</span></h1>
|
<h1>前端的 <span class="gradient-text">SQL 数据库</span></h1>
|
||||||
<p>TypeScript 原生构建,5 种存储引擎,支持完整 SQL 查询。<br>零运行时依赖,开箱即用。AriaEngine 自研引擎:LSM-Tree + WAL + MVCC。</p>
|
<p>TypeScript 原生构建,5 种存储引擎,支持完整 SQL 查询。<br>零运行时依赖,开箱即用。AriaEngine 自研引擎:LSM-Tree + WAL + MVCC。</p>
|
||||||
<div class="actions">
|
<div class="actions">
|
||||||
|
|||||||
+170
-39
@@ -2,13 +2,7 @@
|
|||||||
* AriaEngine — 自研页面式存储引擎主类
|
* AriaEngine — 自研页面式存储引擎主类
|
||||||
* @module engine/aria/index
|
* @module engine/aria/index
|
||||||
*
|
*
|
||||||
* 实现 IStorageEngine 接口。
|
* v0.2.4: 二级索引 + MVCC 集成 + 生产加固
|
||||||
*
|
|
||||||
* v0.2.1: 完整持久化
|
|
||||||
* - Schema 存入 __aria_schemas
|
|
||||||
* - SSTable 元数据存入 __aria_lsm_meta
|
|
||||||
* - WAL 恢复包含行数据
|
|
||||||
* - 启动时自动加载 Schema + SSTable
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { IStorageEngine } from '../interface';
|
import type { IStorageEngine } from '../interface';
|
||||||
@@ -26,6 +20,8 @@ import { WALRecordType, type WALRecord } from './types';
|
|||||||
import { CheckpointManager } from './wal/checkpoint';
|
import { CheckpointManager } from './wal/checkpoint';
|
||||||
import { IndexedDBBackend, MemoryBackend, type IStorageBackend } from './store/backend';
|
import { IndexedDBBackend, MemoryBackend, type IStorageBackend } from './store/backend';
|
||||||
import { OPFSBackend } from './store/opfs_backend';
|
import { OPFSBackend } from './store/opfs_backend';
|
||||||
|
import { MVCCManager } from './transaction/mvcc';
|
||||||
|
import { BloomFilter } from './index/bloom';
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// AriaEngine
|
// AriaEngine
|
||||||
@@ -35,7 +31,7 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
readonly name = 'aria';
|
readonly name = 'aria';
|
||||||
|
|
||||||
private config!: Required<AriaEngineConfig>;
|
private config!: Required<AriaEngineConfig>;
|
||||||
private lsm!: LSM;
|
private lsm!: LSM; // 主键索引 LSM
|
||||||
private wal!: WAL;
|
private wal!: WAL;
|
||||||
private checkpointManager!: CheckpointManager;
|
private checkpointManager!: CheckpointManager;
|
||||||
private backend!: IStorageBackend;
|
private backend!: IStorageBackend;
|
||||||
@@ -47,9 +43,14 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
private tablePKs: Map<string, string> = new Map();
|
private tablePKs: Map<string, string> = new Map();
|
||||||
private opCounter = 0;
|
private opCounter = 0;
|
||||||
|
|
||||||
// 事务
|
// 二级索引:table.colKey → LSM
|
||||||
|
private secondaryIndexes: Map<string, LSM> = new Map();
|
||||||
|
|
||||||
|
// MVCC 事务
|
||||||
|
private mvcc: MVCCManager = new MVCCManager();
|
||||||
private currentTxnId: number | null = null;
|
private currentTxnId: number | null = null;
|
||||||
private txnSnapshot: Map<string, Record<string, unknown>> | null = null;
|
private txnSnapshot: Map<string, Record<string, unknown>> | null = null;
|
||||||
|
private gcCounter = 0;
|
||||||
|
|
||||||
constructor(config: AriaEngineConfig = {}) {
|
constructor(config: AriaEngineConfig = {}) {
|
||||||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||||||
@@ -76,7 +77,7 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
// 2. 构建 SSTableStore
|
// 2. 构建 SSTableStore
|
||||||
const sstableStore = this.createSSTableStore();
|
const sstableStore = this.createSSTableStore();
|
||||||
|
|
||||||
// 3. 初始化 LSM
|
// 3. 初始化主 LSM(PK 索引)
|
||||||
this.lsm = new LSM({
|
this.lsm = new LSM({
|
||||||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
@@ -185,6 +186,25 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
this.schemas.set(schema.name, schema);
|
this.schemas.set(schema.name, schema);
|
||||||
this.tablePKs.set(schema.name, this.getPK(schema));
|
this.tablePKs.set(schema.name, this.getPK(schema));
|
||||||
|
|
||||||
|
// 为索引列创建二级索引 LSM
|
||||||
|
const sstableStore = this.createSSTableStore();
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (colDef.index || colDef.unique || colDef.primaryKey) {
|
||||||
|
const idxKey = `${schema.name}:idx:${colName}`;
|
||||||
|
if (!this.secondaryIndexes.has(idxKey)) {
|
||||||
|
const idxLsm = new LSM({
|
||||||
|
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||||
|
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||||
|
blockSize: this.config.pageSize,
|
||||||
|
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||||||
|
sstableStore,
|
||||||
|
});
|
||||||
|
await idxLsm.init();
|
||||||
|
this.secondaryIndexes.set(idxKey, idxLsm);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
await this.persistSchemas();
|
await this.persistSchemas();
|
||||||
|
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
@@ -263,10 +283,13 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
// Within transaction: buffer to snapshot
|
// Within transaction: buffer to snapshot
|
||||||
this.txnSnapshot.set(key, validated);
|
this.txnSnapshot.set(key, validated);
|
||||||
} else {
|
} else {
|
||||||
// Direct write to LSM
|
// Direct write to LSM (PK index)
|
||||||
this.lsm.put(key, validated);
|
this.lsm.put(key, validated);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, pkValue, validated, null);
|
||||||
|
|
||||||
pks.push(pkValue);
|
pks.push(pkValue);
|
||||||
|
|
||||||
this.wal.append({
|
this.wal.append({
|
||||||
@@ -280,6 +303,7 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
|
|
||||||
this.opCounter += rows.length;
|
this.opCounter += rows.length;
|
||||||
await this.checkpointManager.tick();
|
await this.checkpointManager.tick();
|
||||||
|
this.tryGC();
|
||||||
return pks;
|
return pks;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -373,6 +397,9 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
data: updated,
|
data: updated,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 更新二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), updated, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -407,6 +434,9 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
tableName,
|
tableName,
|
||||||
key: String(row[pkCol]),
|
key: String(row[pkCol]),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 移除二级索引
|
||||||
|
this.updateSecondaryIndexes(tableName, String(row[pkCol]), null, row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -504,34 +534,6 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private tryIndexLookup(
|
|
||||||
tableName: string,
|
|
||||||
query: QueryPlan,
|
|
||||||
): Record<string, unknown>[] | null {
|
|
||||||
if (!query.where) return null;
|
|
||||||
const pkCol = this.tablePKs.get(tableName)!;
|
|
||||||
|
|
||||||
for (const [col, condition] of Object.entries(query.where)) {
|
|
||||||
if (col !== pkCol) continue;
|
|
||||||
|
|
||||||
// 等值条件
|
|
||||||
if (typeof condition !== 'object' || condition === null) {
|
|
||||||
const key = `${tableName}:${condition}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const cond = condition as Record<string, unknown>;
|
|
||||||
if ('$eq' in cond) {
|
|
||||||
const key = `${tableName}:${cond.$eq}`;
|
|
||||||
const value = this.lsm.get(key);
|
|
||||||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private getPK(schema: TableSchema): string {
|
private getPK(schema: TableSchema): string {
|
||||||
for (const [name, col] of Object.entries(schema.columns)) {
|
for (const [name, col] of Object.entries(schema.columns)) {
|
||||||
if (col.primaryKey) return name;
|
if (col.primaryKey) return name;
|
||||||
@@ -686,10 +688,139 @@ export class AriaEngine implements IStorageEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// =======================================================================
|
||||||
|
// 二级索引
|
||||||
|
// =======================================================================
|
||||||
|
|
||||||
|
/** 更新行的二级索引条目 */
|
||||||
|
private updateSecondaryIndexes(
|
||||||
|
tableName: string, pkValue: string,
|
||||||
|
newRow: Record<string, unknown> | null,
|
||||||
|
oldRow: Record<string, unknown> | null,
|
||||||
|
): void {
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema) return;
|
||||||
|
|
||||||
|
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||||
|
if (!colDef.index && !colDef.unique && !colDef.primaryKey) continue;
|
||||||
|
const idxKey = `${tableName}:idx:${colName}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm) continue;
|
||||||
|
|
||||||
|
// 删除旧值
|
||||||
|
if (oldRow) {
|
||||||
|
const oldVal = oldRow[colName];
|
||||||
|
if (oldVal !== undefined && oldVal !== null) {
|
||||||
|
idxLsm.delete(`${String(oldVal)}:${pkValue}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 插入新值
|
||||||
|
if (newRow) {
|
||||||
|
const newVal = newRow[colName];
|
||||||
|
if (newVal !== undefined && newVal !== null) {
|
||||||
|
idxLsm.put(`${String(newVal)}:${pkValue}`, { pk: pkValue });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 通过二级索引快速查找 */
|
||||||
|
private tryIndexLookup(
|
||||||
|
tableName: string,
|
||||||
|
query: QueryPlan,
|
||||||
|
): Record<string, unknown>[] | null {
|
||||||
|
if (!query.where) return null;
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema) return null;
|
||||||
|
const pkCol = this.tablePKs.get(tableName)!;
|
||||||
|
|
||||||
|
for (const [col, condition] of Object.entries(query.where)) {
|
||||||
|
// 跳过 $and/$or/$not 逻辑组合
|
||||||
|
if (col === '$and' || col === '$or' || col === '$not') continue;
|
||||||
|
|
||||||
|
const colDef = schema.columns[col];
|
||||||
|
const hasIndex = colDef && (colDef.index || colDef.unique || colDef.primaryKey);
|
||||||
|
if (!hasIndex && col !== pkCol) continue;
|
||||||
|
|
||||||
|
// PK 等值 → 主 LSM 精确查找
|
||||||
|
if (col === pkCol) {
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
const key = `${tableName}:${condition}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||||||
|
}
|
||||||
|
const cond = condition as Record<string, unknown>;
|
||||||
|
if ('$eq' in cond) {
|
||||||
|
const key = `${tableName}:${cond.$eq}`;
|
||||||
|
const value = this.lsm.get(key);
|
||||||
|
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 二级索引查找
|
||||||
|
const idxKey = `${tableName}:idx:${col}`;
|
||||||
|
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||||
|
if (!idxLsm) continue;
|
||||||
|
|
||||||
|
// $eq → 精确查找
|
||||||
|
if (typeof condition !== 'object' || condition === null) {
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, String(condition), String(condition));
|
||||||
|
}
|
||||||
|
const c = condition as Record<string, unknown>;
|
||||||
|
if ('$eq' in c) {
|
||||||
|
const v = String(c.$eq);
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, v, v);
|
||||||
|
}
|
||||||
|
// $in → 多次精确查找
|
||||||
|
if ('$in' in c && Array.isArray(c.$in)) {
|
||||||
|
const results: Record<string, unknown>[] = [];
|
||||||
|
for (const val of c.$in) {
|
||||||
|
const rows = this.indexScanToRows(tableName, pkCol, idxLsm, col, String(val), String(val));
|
||||||
|
results.push(...rows);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
// $gt / $gte / $lt / $lte → 范围扫描
|
||||||
|
if ('$gt' in c || '$gte' in c || '$lt' in c || '$lte' in c) {
|
||||||
|
const startKey = c.$gt ? `${String(Number(c.$gt) + 1)}:` : c.$gte ? `${String(c.$gte)}:` : `${col}:`;
|
||||||
|
const endKey = c.$lt ? `${String(Number(c.$lt) - 1)}:\uffff` : c.$lte ? `${String(c.$lte)}:\uffff` : `${col}:\uffff`;
|
||||||
|
return this.indexScanToRows(tableName, pkCol, idxLsm, col, startKey, endKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 从索引扫描结果恢复完整行 */
|
||||||
|
private indexScanToRows(
|
||||||
|
tableName: string, pkCol: string, idxLsm: LSM,
|
||||||
|
_col: string, startKey: string, endKey: string,
|
||||||
|
): Record<string, unknown>[] {
|
||||||
|
const entries = idxLsm.rangeScan(startKey, endKey);
|
||||||
|
const rows: Record<string, unknown>[] = [];
|
||||||
|
for (const [, idxEntry] of entries) {
|
||||||
|
const pk = (idxEntry as any).pk as string;
|
||||||
|
if (!pk) continue;
|
||||||
|
const row = this.lsm.get(`${tableName}:${pk}`);
|
||||||
|
if (row) rows.push({ ...row, [pkCol]: pk });
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
// 辅助
|
// 辅助
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
|
||||||
|
/** 每 10 次 gc 计数器触发一次 MVCC 垃圾回收 */
|
||||||
|
private tryGC(): void {
|
||||||
|
this.gcCounter++;
|
||||||
|
if (this.gcCounter >= 10) {
|
||||||
|
this.mvcc.gc(100);
|
||||||
|
this.gcCounter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private ensureOpen(): void {
|
private ensureOpen(): void {
|
||||||
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import type { IndexEntry, SSTableMeta } from '../types';
|
import type { IndexEntry, SSTableMeta } from '../types';
|
||||||
|
import { BloomFilter } from './bloom';
|
||||||
|
|
||||||
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
|
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ export class SSTableReader {
|
|||||||
private indexEntries: IndexEntry[] = [];
|
private indexEntries: IndexEntry[] = [];
|
||||||
private entryCount = 0;
|
private entryCount = 0;
|
||||||
private meta: SSTableMeta;
|
private meta: SSTableMeta;
|
||||||
|
private bloomFilter: BloomFilter | null = null;
|
||||||
|
|
||||||
constructor(data: Uint8Array, meta: SSTableMeta) {
|
constructor(data: Uint8Array, meta: SSTableMeta) {
|
||||||
this.data = data;
|
this.data = data;
|
||||||
@@ -31,6 +33,9 @@ export class SSTableReader {
|
|||||||
|
|
||||||
/** 精确查找 key */
|
/** 精确查找 key */
|
||||||
get(targetKey: string): Record<string, unknown> | null {
|
get(targetKey: string): Record<string, unknown> | null {
|
||||||
|
// Bloom Filter 快速否定
|
||||||
|
if (this.bloomFilter && !this.bloomFilter.mayContain(targetKey)) return null;
|
||||||
|
|
||||||
const blockIdx = this.locateBlock(targetKey);
|
const blockIdx = this.locateBlock(targetKey);
|
||||||
if (blockIdx < 0) return null;
|
if (blockIdx < 0) return null;
|
||||||
|
|
||||||
@@ -175,10 +180,23 @@ export class SSTableReader {
|
|||||||
|
|
||||||
const indexOffset = this.view.getUint32(footerOffset, false);
|
const indexOffset = this.view.getUint32(footerOffset, false);
|
||||||
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
||||||
|
const bloomOffset = this.view.getUint32(footerOffset + 8, false);
|
||||||
|
const bloomSize = this.view.getUint32(footerOffset + 12, false);
|
||||||
|
const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
|
||||||
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
||||||
|
|
||||||
// 解析索引块
|
// 解析索引块
|
||||||
this.parseIndexBlock(indexOffset, indexSize);
|
this.parseIndexBlock(indexOffset, indexSize);
|
||||||
|
|
||||||
|
// 加载 Bloom Filter
|
||||||
|
if (bloomOffset > 0 && bloomSize > 0 && bloomOffset + bloomSize <= this.data.byteLength) {
|
||||||
|
try {
|
||||||
|
const bloomBytes = this.data.slice(bloomOffset, bloomOffset + bloomSize);
|
||||||
|
this.bloomFilter = BloomFilter.fromData(bloomBytes, bloomHashCount || 10);
|
||||||
|
} catch {
|
||||||
|
// 损坏的 bloom filter 不影响读取(仅跳过快速否定优化)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private parseIndexBlock(offset: number, _size: number): void {
|
private parseIndexBlock(offset: number, _size: number): void {
|
||||||
|
|||||||
@@ -92,8 +92,12 @@ export class SSTableBuilder {
|
|||||||
|
|
||||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||||
|
|
||||||
// 写入到 buffer
|
// 序列化 bloom filter 以获取其大小
|
||||||
const finalSize = totalSize + indexBlockSize + SSTABLE_FOOTER_SIZE;
|
const bloomData = bloomFilter.serialize();
|
||||||
|
const bloomSize = bloomData.byteLength;
|
||||||
|
|
||||||
|
// 写入到 buffer(包含 bloom block)
|
||||||
|
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
|
||||||
const buf = new ArrayBuffer(finalSize);
|
const buf = new ArrayBuffer(finalSize);
|
||||||
const view = new DataView(buf);
|
const view = new DataView(buf);
|
||||||
|
|
||||||
@@ -108,12 +112,17 @@ export class SSTableBuilder {
|
|||||||
const indexOffset = offset;
|
const indexOffset = offset;
|
||||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||||
|
|
||||||
|
// ---- Bloom Filter Block ----
|
||||||
|
const bloomOffset = offset;
|
||||||
|
new Uint8Array(view.buffer).set(bloomData, offset);
|
||||||
|
offset += bloomSize;
|
||||||
|
|
||||||
// ---- Footer ----
|
// ---- Footer ----
|
||||||
const footerOffset = offset;
|
const footerOffset = offset;
|
||||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||||
view.setUint32(footerOffset + 8, 0, false); // bloom_offset (embedded in footer)
|
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
|
||||||
view.setUint32(footerOffset + 12, 0, false); // bloom_size
|
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
|
||||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC, false);
|
view.setUint32(footerOffset + 24, SSTABLE_MAGIC, false);
|
||||||
|
|||||||
@@ -280,6 +280,10 @@ export interface AriaEngineConfig {
|
|||||||
compression?: boolean;
|
compression?: boolean;
|
||||||
/** 存储后端 */
|
/** 存储后端 */
|
||||||
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
||||||
|
/** WAL 大小阈值(字节,超过则强制 checkpoint,默认 16MB) */
|
||||||
|
walSizeThreshold?: number;
|
||||||
|
/** 最大内存预算(MB,默认 64) */
|
||||||
|
maxMemoryMB?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DEFAULT_ARIA_CONFIG: Required<AriaEngineConfig> = {
|
export const DEFAULT_ARIA_CONFIG: Required<AriaEngineConfig> = {
|
||||||
@@ -289,8 +293,10 @@ export const DEFAULT_ARIA_CONFIG: Required<AriaEngineConfig> = {
|
|||||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||||
walEnabled: true,
|
walEnabled: true,
|
||||||
walSyncMode: 'batch',
|
walSyncMode: 'full',
|
||||||
checkpointInterval: 1000,
|
checkpointInterval: 1000,
|
||||||
compression: false,
|
compression: false,
|
||||||
storageBackend: 'indexeddb',
|
storageBackend: 'indexeddb',
|
||||||
|
walSizeThreshold: 16 * 1024 * 1024, // 16MB
|
||||||
|
maxMemoryMB: 64,
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user