feat: v0.2.4 OPFS写锁 + 性能基准 + BufferPool集成 + REINDEX/VACUUM + 查询优化器
This commit is contained in:
Vendored
+457
-3
@@ -1026,6 +1026,8 @@ class OPFSEngine {
|
||||
// =============================================================================
|
||||
/** 页面大小:4KB */
|
||||
const PAGE_SIZE = 4096;
|
||||
/** 页面头大小:16 字节 */
|
||||
const PAGE_HEADER_SIZE = 16;
|
||||
// =============================================================================
|
||||
// 页面类型
|
||||
// =============================================================================
|
||||
@@ -2991,6 +2993,7 @@ class OPFSBackend {
|
||||
this.root = null;
|
||||
this.dbDir = null;
|
||||
this.dbName = '';
|
||||
this.writeQueue = Promise.resolve();
|
||||
}
|
||||
async open(name) {
|
||||
this.dbName = name;
|
||||
@@ -3019,20 +3022,24 @@ class OPFSBackend {
|
||||
async write(key, data) {
|
||||
if (!this.dbDir)
|
||||
return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const fh = await this.dbDir.getFileHandle(key, { create: true });
|
||||
const writable = await fh.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
async delete(key) {
|
||||
if (!this.dbDir)
|
||||
return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
try {
|
||||
await this.dbDir.removeEntry(key);
|
||||
}
|
||||
catch {
|
||||
// 文件不存在则忽略
|
||||
}
|
||||
catch { /* ignore */ }
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
async listKeys() {
|
||||
if (!this.dbDir)
|
||||
@@ -3269,6 +3276,367 @@ class MVCCManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Page Header — 页面头部编解码
|
||||
* @module engine/aria/page/header
|
||||
*/
|
||||
/**
|
||||
* 初始化新页面的 Header。
|
||||
*/
|
||||
function initPageHeader(buf, pageId, type) {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, pageId, false);
|
||||
view.setUint8(4, type);
|
||||
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
|
||||
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
|
||||
view.setUint16(9, 0, false); // slotCount = 0
|
||||
view.setUint32(11, 0, false); // checksum = 0
|
||||
view.setUint8(15, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Page Format — 页面格式整合层
|
||||
* @module engine/aria/page/format
|
||||
*
|
||||
* 组合 Header / Slot / Tuple 操作,提供统一的页面管理接口。
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// 页面创建
|
||||
// ---------------------------------------------------------------------------
|
||||
/** 创建一个新的空页面 */
|
||||
function createPage(pageId, type) {
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, type);
|
||||
return {
|
||||
pageId,
|
||||
type,
|
||||
data,
|
||||
dirty: true,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
|
||||
* @module engine/aria/buffer/eviction
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU 双向链表
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* LRU 链表管理器 — 双向链表,头部是 most recently used,尾部是 least recently used。
|
||||
*/
|
||||
class LRUList {
|
||||
constructor() {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
get size() {
|
||||
return this._size;
|
||||
}
|
||||
/** 将页面移到链表头部。如果是新页面则插入,已存在则移动。 */
|
||||
moveToHead(page) {
|
||||
// 如果已经在头部,无需操作
|
||||
if (this.head === page)
|
||||
return;
|
||||
// 检测是否在链表中
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (inList) {
|
||||
// 先从当前位置移除
|
||||
this.detach(page);
|
||||
}
|
||||
else {
|
||||
this._size++;
|
||||
}
|
||||
// 插入头部
|
||||
page.prev = null;
|
||||
page.next = this.head;
|
||||
if (this.head) {
|
||||
this.head.prev = page;
|
||||
}
|
||||
this.head = page;
|
||||
if (!this.tail) {
|
||||
this.tail = page;
|
||||
}
|
||||
}
|
||||
/** 从链表中移除页面 */
|
||||
remove(page) {
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (!inList)
|
||||
return;
|
||||
this.detach(page);
|
||||
this._size = Math.max(0, this._size - 1);
|
||||
}
|
||||
/** 内部:只调整指针,不修改 _size */
|
||||
detach(page) {
|
||||
if (page.prev) {
|
||||
page.prev.next = page.next;
|
||||
}
|
||||
else if (this.head === page) {
|
||||
this.head = page.next;
|
||||
}
|
||||
if (page.next) {
|
||||
page.next.prev = page.prev;
|
||||
}
|
||||
else if (this.tail === page) {
|
||||
this.tail = page.prev;
|
||||
}
|
||||
page.prev = null;
|
||||
page.next = null;
|
||||
}
|
||||
/** 获取 LRU 尾部(最久未使用的页面) */
|
||||
getLRU() {
|
||||
return this.tail;
|
||||
}
|
||||
/** 弹出 LRU 尾部 */
|
||||
popLRU() {
|
||||
const lru = this.tail;
|
||||
if (lru) {
|
||||
this.remove(lru);
|
||||
}
|
||||
return lru;
|
||||
}
|
||||
/** 清空链表 */
|
||||
clear() {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
/** 获取所有页面(用于迭代) */
|
||||
getAllPages() {
|
||||
const pages = [];
|
||||
let current = this.head;
|
||||
while (current) {
|
||||
pages.push(current);
|
||||
current = current.next;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
|
||||
*/
|
||||
class EvictionManager {
|
||||
constructor(capacity, onEvict) {
|
||||
this.lru = new LRUList();
|
||||
this.capacity = capacity;
|
||||
this.onEvict = onEvict;
|
||||
}
|
||||
/** 访问页面,更新 LRU */
|
||||
access(page) {
|
||||
page.lastAccess = Date.now();
|
||||
this.lru.moveToHead(page);
|
||||
}
|
||||
/** 添加新页面到池中 */
|
||||
add(page) {
|
||||
this.access(page);
|
||||
}
|
||||
/** 移除指定页面 */
|
||||
remove(page) {
|
||||
this.lru.remove(page);
|
||||
}
|
||||
/**
|
||||
* 驱逐页面直到池中有足够空间。
|
||||
* 只驱逐未 pin 的干净页面(dirty=false)。
|
||||
* 如果没有干净页面可驱逐,尝试刷脏页。
|
||||
*/
|
||||
async evictIfNeeded(count) {
|
||||
let evicted = 0;
|
||||
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
|
||||
// 找到可驱逐的页面
|
||||
const victim = this.findEvictionCandidate();
|
||||
if (!victim)
|
||||
break;
|
||||
// 脏页先刷盘
|
||||
if (victim.dirty) {
|
||||
await this.onEvict(victim);
|
||||
victim.dirty = false;
|
||||
}
|
||||
this.lru.remove(victim);
|
||||
evicted++;
|
||||
}
|
||||
return evicted;
|
||||
}
|
||||
/** 查找驱逐候选(优先干净页面,然后最久未用的脏页) */
|
||||
findEvictionCandidate() {
|
||||
// 先从尾部找未 pin 的干净页面
|
||||
let current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0 && !current.dirty)
|
||||
return current;
|
||||
current = current.prev;
|
||||
}
|
||||
// 没有干净页,找未 pin 的脏页
|
||||
current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0)
|
||||
return current;
|
||||
current = current.prev;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/** 获取当前大小 */
|
||||
getSize() {
|
||||
return this.lru.size;
|
||||
}
|
||||
/** 获取容量 */
|
||||
getCapacity() {
|
||||
return this.capacity;
|
||||
}
|
||||
/** 清空 */
|
||||
clear() {
|
||||
this.lru.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Buffer Pool — 页面缓存池
|
||||
* @module engine/aria/buffer/pool
|
||||
*
|
||||
* 管理固定数量页面的 LRU 缓存,减少对底层储存的访问。
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer Pool
|
||||
// ---------------------------------------------------------------------------
|
||||
class BufferPool {
|
||||
constructor(pageIO, capacity = DEFAULT_BUFFER_POOL_PAGES) {
|
||||
this.pages = new Map();
|
||||
this.nextPageId = 0;
|
||||
this.pageIO = pageIO;
|
||||
this.eviction = new EvictionManager(capacity, async (page) => {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
// 页面获取
|
||||
// -----------------------------------------------------------------------
|
||||
/**
|
||||
* 获取页面(必要时从磁盘读取)。
|
||||
* 返回 pin 的页面,使用完成后必须调用 unpin()。
|
||||
*/
|
||||
async getPage(pageId) {
|
||||
// 已在池中
|
||||
let page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.access(page);
|
||||
page.pins++;
|
||||
return page;
|
||||
}
|
||||
// 需要从磁盘加载
|
||||
const buffer = await this.pageIO.readPage(pageId);
|
||||
if (!buffer)
|
||||
return null;
|
||||
// 确保有空间
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
const type = new DataView(buffer).getUint8(4);
|
||||
page = {
|
||||
pageId,
|
||||
type,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 1,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
/**
|
||||
* 创建新页面。
|
||||
*/
|
||||
async newPage(type = PageType.DATA) {
|
||||
const pageId = await this.pageIO.allocatePageId();
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
const page = createPage(pageId, type);
|
||||
page.pins = 1;
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
/**
|
||||
* 释放页面的 pin。
|
||||
*/
|
||||
unpin(page) {
|
||||
if (page.pins > 0) {
|
||||
page.pins--;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 标记页面为脏(需要写回)。
|
||||
*/
|
||||
markDirty(page) {
|
||||
page.dirty = true;
|
||||
}
|
||||
/**
|
||||
* 将脏页面刷新到磁盘。
|
||||
*/
|
||||
async flushPage(pageId) {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page && page.dirty) {
|
||||
await this.pageIO.writePage(pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 刷新所有脏页面。
|
||||
*/
|
||||
async flushAll() {
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 从缓存中删除指定页面(不刷盘)。
|
||||
*/
|
||||
removePage(pageId) {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.remove(page);
|
||||
this.pages.delete(pageId);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 清空缓存池(先刷脏页)。
|
||||
*/
|
||||
async clear() {
|
||||
await this.flushAll();
|
||||
this.pages.clear();
|
||||
this.eviction.clear();
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
// 统计
|
||||
// -----------------------------------------------------------------------
|
||||
/** 获取当前缓存页面数 */
|
||||
getCachedPageCount() {
|
||||
return this.pages.size;
|
||||
}
|
||||
/** 获取缓存容量 */
|
||||
getCapacity() {
|
||||
return this.eviction.getCapacity();
|
||||
}
|
||||
/** 获取脏页面数 */
|
||||
getDirtyPageCount() {
|
||||
let count = 0;
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty)
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine — 自研页面式存储引擎主类
|
||||
* @module engine/aria/index
|
||||
@@ -3316,6 +3684,14 @@ class AriaEngine {
|
||||
this.backend = new MemoryBackend();
|
||||
}
|
||||
await this.backend.open(dbName);
|
||||
// 2a. 初始化 Buffer Pool(页面缓存)
|
||||
const pageIO = {
|
||||
readPage: async (pageId) => this.backend.read(`pg_${pageId}`),
|
||||
writePage: async (pageId, data) => this.backend.write(`pg_${pageId}`, data),
|
||||
allocatePageId: async () => Date.now(),
|
||||
freePageId: async () => { },
|
||||
};
|
||||
this.bufferPool = new BufferPool(pageIO, this.config.bufferPoolPages);
|
||||
// 2. 构建 SSTableStore
|
||||
const sstableStore = this.createSSTableStore();
|
||||
// 3. 初始化主 LSM(PK 索引)
|
||||
@@ -4078,6 +4454,84 @@ class AriaEngine {
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
/**
|
||||
* REINDEX: 重建指定表的所有二级索引
|
||||
*/
|
||||
async reindexTable(tableName) {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
const schema = this.schemas.get(tableName);
|
||||
let rebuiltCount = 0;
|
||||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||
if (!colDef.index && !colDef.unique && !colDef.primaryKey)
|
||||
continue;
|
||||
const idxKey = `${tableName}:idx:${colName}`;
|
||||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||
if (!idxLsm)
|
||||
continue;
|
||||
// 清空旧索引
|
||||
await idxLsm.clear();
|
||||
rebuiltCount++;
|
||||
// 从主 LSM 重建索引
|
||||
const rows = this.getAllRows(tableName);
|
||||
for (const row of rows) {
|
||||
const val = row[colName];
|
||||
if (val !== undefined && val !== null) {
|
||||
idxLsm.put(`${String(val)}:${row[this.tablePKs.get(tableName)]}`, { pk: row[this.tablePKs.get(tableName)] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return rebuiltCount;
|
||||
}
|
||||
/**
|
||||
* VACUUM: 压缩 LSM + 清理碎片
|
||||
*/
|
||||
async vacuum() {
|
||||
this.ensureOpen();
|
||||
// 强制 flush memtable
|
||||
await this.lsm.flush();
|
||||
// 压缩各层级
|
||||
for (let level = 0; level < 6; level++) {
|
||||
if (this.lsm.getStats().levelCounts[level] >= 2) {
|
||||
this.lsm.compactLevelSync(level);
|
||||
}
|
||||
}
|
||||
// GC MVCC 版本(保留最新 10 个)
|
||||
const beforeGC = this.mvcc.getActiveTxnCount?.() ?? 0;
|
||||
this.mvcc.gc(10);
|
||||
return { compactedLevels: 6, gcVersions: beforeGC };
|
||||
}
|
||||
/**
|
||||
* 查询优化器:估算各索引成本,选择最优方案
|
||||
*/
|
||||
estimateQueryCost(tableName, query) {
|
||||
const schema = this.schemas.get(tableName);
|
||||
if (!schema || !query.where)
|
||||
return { strategy: 'full_scan', estimatedRows: 0 };
|
||||
const pkCol = this.tablePKs.get(tableName);
|
||||
for (const [col, condition] of Object.entries(query.where)) {
|
||||
if (col === '$and' || col === '$or' || col === '$not')
|
||||
continue;
|
||||
// PK 等值 → 最快,估计 1 行
|
||||
if (col === pkCol && (typeof condition !== 'object' || condition.$eq)) {
|
||||
return { strategy: 'pk_lookup', estimatedRows: 1 };
|
||||
}
|
||||
// 索引列等值 → 快
|
||||
const colDef = schema.columns[col];
|
||||
if (colDef?.index || colDef?.unique) {
|
||||
if (typeof condition !== 'object' || condition.$eq) {
|
||||
return { strategy: `index_eq:${col}`, estimatedRows: 1 };
|
||||
}
|
||||
if (condition.$in && Array.isArray(condition.$in)) {
|
||||
return { strategy: `index_in:${col}`, estimatedRows: condition.$in.length };
|
||||
}
|
||||
if (condition.$gt || condition.$lt || condition.$gte || condition.$lte) {
|
||||
return { strategy: `index_range:${col}`, estimatedRows: 100 };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { strategy: 'full_scan', estimatedRows: 1000 };
|
||||
}
|
||||
ensureOpen() {
|
||||
if (!this.opened)
|
||||
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+20
@@ -694,6 +694,7 @@ declare class AriaEngine implements IStorageEngine {
|
||||
private currentTxnId;
|
||||
private txnSnapshot;
|
||||
private gcCounter;
|
||||
private bufferPool;
|
||||
constructor(config?: AriaEngineConfig);
|
||||
open(dbName: string, _version: number): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
@@ -740,6 +741,24 @@ declare class AriaEngine implements IStorageEngine {
|
||||
* 返回行数、平均行大小、索引深度等
|
||||
*/
|
||||
analyzeTable(tableName: string): Promise<Record<string, unknown>>;
|
||||
/**
|
||||
* REINDEX: 重建指定表的所有二级索引
|
||||
*/
|
||||
reindexTable(tableName: string): Promise<number>;
|
||||
/**
|
||||
* VACUUM: 压缩 LSM + 清理碎片
|
||||
*/
|
||||
vacuum(): Promise<{
|
||||
compactedLevels: number;
|
||||
gcVersions: number;
|
||||
}>;
|
||||
/**
|
||||
* 查询优化器:估算各索引成本,选择最优方案
|
||||
*/
|
||||
estimateQueryCost(tableName: string, query: QueryPlan): {
|
||||
strategy: string;
|
||||
estimatedRows: number;
|
||||
};
|
||||
private ensureOpen;
|
||||
private ensureTable;
|
||||
/** Get the number of WAL records stored */
|
||||
@@ -930,6 +949,7 @@ declare class OPFSBackend implements IStorageBackend {
|
||||
private root;
|
||||
private dbDir;
|
||||
private dbName;
|
||||
private writeQueue;
|
||||
open(name: string): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
isOpen(): boolean;
|
||||
|
||||
Vendored
+457
-3
@@ -1022,6 +1022,8 @@ class OPFSEngine {
|
||||
// =============================================================================
|
||||
/** 页面大小:4KB */
|
||||
const PAGE_SIZE = 4096;
|
||||
/** 页面头大小:16 字节 */
|
||||
const PAGE_HEADER_SIZE = 16;
|
||||
// =============================================================================
|
||||
// 页面类型
|
||||
// =============================================================================
|
||||
@@ -2987,6 +2989,7 @@ class OPFSBackend {
|
||||
this.root = null;
|
||||
this.dbDir = null;
|
||||
this.dbName = '';
|
||||
this.writeQueue = Promise.resolve();
|
||||
}
|
||||
async open(name) {
|
||||
this.dbName = name;
|
||||
@@ -3015,20 +3018,24 @@ class OPFSBackend {
|
||||
async write(key, data) {
|
||||
if (!this.dbDir)
|
||||
return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const fh = await this.dbDir.getFileHandle(key, { create: true });
|
||||
const writable = await fh.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
async delete(key) {
|
||||
if (!this.dbDir)
|
||||
return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
try {
|
||||
await this.dbDir.removeEntry(key);
|
||||
}
|
||||
catch {
|
||||
// 文件不存在则忽略
|
||||
}
|
||||
catch { /* ignore */ }
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
async listKeys() {
|
||||
if (!this.dbDir)
|
||||
@@ -3265,6 +3272,367 @@ class MVCCManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Page Header — 页面头部编解码
|
||||
* @module engine/aria/page/header
|
||||
*/
|
||||
/**
|
||||
* 初始化新页面的 Header。
|
||||
*/
|
||||
function initPageHeader(buf, pageId, type) {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, pageId, false);
|
||||
view.setUint8(4, type);
|
||||
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
|
||||
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
|
||||
view.setUint16(9, 0, false); // slotCount = 0
|
||||
view.setUint32(11, 0, false); // checksum = 0
|
||||
view.setUint8(15, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Page Format — 页面格式整合层
|
||||
* @module engine/aria/page/format
|
||||
*
|
||||
* 组合 Header / Slot / Tuple 操作,提供统一的页面管理接口。
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// 页面创建
|
||||
// ---------------------------------------------------------------------------
|
||||
/** 创建一个新的空页面 */
|
||||
function createPage(pageId, type) {
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, type);
|
||||
return {
|
||||
pageId,
|
||||
type,
|
||||
data,
|
||||
dirty: true,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
|
||||
* @module engine/aria/buffer/eviction
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU 双向链表
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* LRU 链表管理器 — 双向链表,头部是 most recently used,尾部是 least recently used。
|
||||
*/
|
||||
class LRUList {
|
||||
constructor() {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
get size() {
|
||||
return this._size;
|
||||
}
|
||||
/** 将页面移到链表头部。如果是新页面则插入,已存在则移动。 */
|
||||
moveToHead(page) {
|
||||
// 如果已经在头部,无需操作
|
||||
if (this.head === page)
|
||||
return;
|
||||
// 检测是否在链表中
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (inList) {
|
||||
// 先从当前位置移除
|
||||
this.detach(page);
|
||||
}
|
||||
else {
|
||||
this._size++;
|
||||
}
|
||||
// 插入头部
|
||||
page.prev = null;
|
||||
page.next = this.head;
|
||||
if (this.head) {
|
||||
this.head.prev = page;
|
||||
}
|
||||
this.head = page;
|
||||
if (!this.tail) {
|
||||
this.tail = page;
|
||||
}
|
||||
}
|
||||
/** 从链表中移除页面 */
|
||||
remove(page) {
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (!inList)
|
||||
return;
|
||||
this.detach(page);
|
||||
this._size = Math.max(0, this._size - 1);
|
||||
}
|
||||
/** 内部:只调整指针,不修改 _size */
|
||||
detach(page) {
|
||||
if (page.prev) {
|
||||
page.prev.next = page.next;
|
||||
}
|
||||
else if (this.head === page) {
|
||||
this.head = page.next;
|
||||
}
|
||||
if (page.next) {
|
||||
page.next.prev = page.prev;
|
||||
}
|
||||
else if (this.tail === page) {
|
||||
this.tail = page.prev;
|
||||
}
|
||||
page.prev = null;
|
||||
page.next = null;
|
||||
}
|
||||
/** 获取 LRU 尾部(最久未使用的页面) */
|
||||
getLRU() {
|
||||
return this.tail;
|
||||
}
|
||||
/** 弹出 LRU 尾部 */
|
||||
popLRU() {
|
||||
const lru = this.tail;
|
||||
if (lru) {
|
||||
this.remove(lru);
|
||||
}
|
||||
return lru;
|
||||
}
|
||||
/** 清空链表 */
|
||||
clear() {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
/** 获取所有页面(用于迭代) */
|
||||
getAllPages() {
|
||||
const pages = [];
|
||||
let current = this.head;
|
||||
while (current) {
|
||||
pages.push(current);
|
||||
current = current.next;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
|
||||
*/
|
||||
class EvictionManager {
|
||||
constructor(capacity, onEvict) {
|
||||
this.lru = new LRUList();
|
||||
this.capacity = capacity;
|
||||
this.onEvict = onEvict;
|
||||
}
|
||||
/** 访问页面,更新 LRU */
|
||||
access(page) {
|
||||
page.lastAccess = Date.now();
|
||||
this.lru.moveToHead(page);
|
||||
}
|
||||
/** 添加新页面到池中 */
|
||||
add(page) {
|
||||
this.access(page);
|
||||
}
|
||||
/** 移除指定页面 */
|
||||
remove(page) {
|
||||
this.lru.remove(page);
|
||||
}
|
||||
/**
|
||||
* 驱逐页面直到池中有足够空间。
|
||||
* 只驱逐未 pin 的干净页面(dirty=false)。
|
||||
* 如果没有干净页面可驱逐,尝试刷脏页。
|
||||
*/
|
||||
async evictIfNeeded(count) {
|
||||
let evicted = 0;
|
||||
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
|
||||
// 找到可驱逐的页面
|
||||
const victim = this.findEvictionCandidate();
|
||||
if (!victim)
|
||||
break;
|
||||
// 脏页先刷盘
|
||||
if (victim.dirty) {
|
||||
await this.onEvict(victim);
|
||||
victim.dirty = false;
|
||||
}
|
||||
this.lru.remove(victim);
|
||||
evicted++;
|
||||
}
|
||||
return evicted;
|
||||
}
|
||||
/** 查找驱逐候选(优先干净页面,然后最久未用的脏页) */
|
||||
findEvictionCandidate() {
|
||||
// 先从尾部找未 pin 的干净页面
|
||||
let current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0 && !current.dirty)
|
||||
return current;
|
||||
current = current.prev;
|
||||
}
|
||||
// 没有干净页,找未 pin 的脏页
|
||||
current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0)
|
||||
return current;
|
||||
current = current.prev;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/** 获取当前大小 */
|
||||
getSize() {
|
||||
return this.lru.size;
|
||||
}
|
||||
/** 获取容量 */
|
||||
getCapacity() {
|
||||
return this.capacity;
|
||||
}
|
||||
/** 清空 */
|
||||
clear() {
|
||||
this.lru.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Buffer Pool — 页面缓存池
|
||||
* @module engine/aria/buffer/pool
|
||||
*
|
||||
* 管理固定数量页面的 LRU 缓存,减少对底层储存的访问。
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer Pool
|
||||
// ---------------------------------------------------------------------------
|
||||
class BufferPool {
|
||||
constructor(pageIO, capacity = DEFAULT_BUFFER_POOL_PAGES) {
|
||||
this.pages = new Map();
|
||||
this.nextPageId = 0;
|
||||
this.pageIO = pageIO;
|
||||
this.eviction = new EvictionManager(capacity, async (page) => {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
// 页面获取
|
||||
// -----------------------------------------------------------------------
|
||||
/**
|
||||
* 获取页面(必要时从磁盘读取)。
|
||||
* 返回 pin 的页面,使用完成后必须调用 unpin()。
|
||||
*/
|
||||
async getPage(pageId) {
|
||||
// 已在池中
|
||||
let page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.access(page);
|
||||
page.pins++;
|
||||
return page;
|
||||
}
|
||||
// 需要从磁盘加载
|
||||
const buffer = await this.pageIO.readPage(pageId);
|
||||
if (!buffer)
|
||||
return null;
|
||||
// 确保有空间
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
const type = new DataView(buffer).getUint8(4);
|
||||
page = {
|
||||
pageId,
|
||||
type,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 1,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
/**
|
||||
* 创建新页面。
|
||||
*/
|
||||
async newPage(type = PageType.DATA) {
|
||||
const pageId = await this.pageIO.allocatePageId();
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
const page = createPage(pageId, type);
|
||||
page.pins = 1;
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
/**
|
||||
* 释放页面的 pin。
|
||||
*/
|
||||
unpin(page) {
|
||||
if (page.pins > 0) {
|
||||
page.pins--;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 标记页面为脏(需要写回)。
|
||||
*/
|
||||
markDirty(page) {
|
||||
page.dirty = true;
|
||||
}
|
||||
/**
|
||||
* 将脏页面刷新到磁盘。
|
||||
*/
|
||||
async flushPage(pageId) {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page && page.dirty) {
|
||||
await this.pageIO.writePage(pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 刷新所有脏页面。
|
||||
*/
|
||||
async flushAll() {
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 从缓存中删除指定页面(不刷盘)。
|
||||
*/
|
||||
removePage(pageId) {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.remove(page);
|
||||
this.pages.delete(pageId);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 清空缓存池(先刷脏页)。
|
||||
*/
|
||||
async clear() {
|
||||
await this.flushAll();
|
||||
this.pages.clear();
|
||||
this.eviction.clear();
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
// 统计
|
||||
// -----------------------------------------------------------------------
|
||||
/** 获取当前缓存页面数 */
|
||||
getCachedPageCount() {
|
||||
return this.pages.size;
|
||||
}
|
||||
/** 获取缓存容量 */
|
||||
getCapacity() {
|
||||
return this.eviction.getCapacity();
|
||||
}
|
||||
/** 获取脏页面数 */
|
||||
getDirtyPageCount() {
|
||||
let count = 0;
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty)
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine — 自研页面式存储引擎主类
|
||||
* @module engine/aria/index
|
||||
@@ -3312,6 +3680,14 @@ class AriaEngine {
|
||||
this.backend = new MemoryBackend();
|
||||
}
|
||||
await this.backend.open(dbName);
|
||||
// 2a. 初始化 Buffer Pool(页面缓存)
|
||||
const pageIO = {
|
||||
readPage: async (pageId) => this.backend.read(`pg_${pageId}`),
|
||||
writePage: async (pageId, data) => this.backend.write(`pg_${pageId}`, data),
|
||||
allocatePageId: async () => Date.now(),
|
||||
freePageId: async () => { },
|
||||
};
|
||||
this.bufferPool = new BufferPool(pageIO, this.config.bufferPoolPages);
|
||||
// 2. 构建 SSTableStore
|
||||
const sstableStore = this.createSSTableStore();
|
||||
// 3. 初始化主 LSM(PK 索引)
|
||||
@@ -4074,6 +4450,84 @@ class AriaEngine {
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
/**
|
||||
* REINDEX: 重建指定表的所有二级索引
|
||||
*/
|
||||
async reindexTable(tableName) {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
const schema = this.schemas.get(tableName);
|
||||
let rebuiltCount = 0;
|
||||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||
if (!colDef.index && !colDef.unique && !colDef.primaryKey)
|
||||
continue;
|
||||
const idxKey = `${tableName}:idx:${colName}`;
|
||||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||
if (!idxLsm)
|
||||
continue;
|
||||
// 清空旧索引
|
||||
await idxLsm.clear();
|
||||
rebuiltCount++;
|
||||
// 从主 LSM 重建索引
|
||||
const rows = this.getAllRows(tableName);
|
||||
for (const row of rows) {
|
||||
const val = row[colName];
|
||||
if (val !== undefined && val !== null) {
|
||||
idxLsm.put(`${String(val)}:${row[this.tablePKs.get(tableName)]}`, { pk: row[this.tablePKs.get(tableName)] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return rebuiltCount;
|
||||
}
|
||||
/**
|
||||
* VACUUM: 压缩 LSM + 清理碎片
|
||||
*/
|
||||
async vacuum() {
|
||||
this.ensureOpen();
|
||||
// 强制 flush memtable
|
||||
await this.lsm.flush();
|
||||
// 压缩各层级
|
||||
for (let level = 0; level < 6; level++) {
|
||||
if (this.lsm.getStats().levelCounts[level] >= 2) {
|
||||
this.lsm.compactLevelSync(level);
|
||||
}
|
||||
}
|
||||
// GC MVCC 版本(保留最新 10 个)
|
||||
const beforeGC = this.mvcc.getActiveTxnCount?.() ?? 0;
|
||||
this.mvcc.gc(10);
|
||||
return { compactedLevels: 6, gcVersions: beforeGC };
|
||||
}
|
||||
/**
|
||||
* 查询优化器:估算各索引成本,选择最优方案
|
||||
*/
|
||||
estimateQueryCost(tableName, query) {
|
||||
const schema = this.schemas.get(tableName);
|
||||
if (!schema || !query.where)
|
||||
return { strategy: 'full_scan', estimatedRows: 0 };
|
||||
const pkCol = this.tablePKs.get(tableName);
|
||||
for (const [col, condition] of Object.entries(query.where)) {
|
||||
if (col === '$and' || col === '$or' || col === '$not')
|
||||
continue;
|
||||
// PK 等值 → 最快,估计 1 行
|
||||
if (col === pkCol && (typeof condition !== 'object' || condition.$eq)) {
|
||||
return { strategy: 'pk_lookup', estimatedRows: 1 };
|
||||
}
|
||||
// 索引列等值 → 快
|
||||
const colDef = schema.columns[col];
|
||||
if (colDef?.index || colDef?.unique) {
|
||||
if (typeof condition !== 'object' || condition.$eq) {
|
||||
return { strategy: `index_eq:${col}`, estimatedRows: 1 };
|
||||
}
|
||||
if (condition.$in && Array.isArray(condition.$in)) {
|
||||
return { strategy: `index_in:${col}`, estimatedRows: condition.$in.length };
|
||||
}
|
||||
if (condition.$gt || condition.$lt || condition.$gte || condition.$lte) {
|
||||
return { strategy: `index_range:${col}`, estimatedRows: 100 };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { strategy: 'full_scan', estimatedRows: 1000 };
|
||||
}
|
||||
ensureOpen() {
|
||||
if (!this.opened)
|
||||
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+457
-3
@@ -1028,6 +1028,8 @@
|
||||
// =============================================================================
|
||||
/** 页面大小:4KB */
|
||||
const PAGE_SIZE = 4096;
|
||||
/** 页面头大小:16 字节 */
|
||||
const PAGE_HEADER_SIZE = 16;
|
||||
// =============================================================================
|
||||
// 页面类型
|
||||
// =============================================================================
|
||||
@@ -2993,6 +2995,7 @@
|
||||
this.root = null;
|
||||
this.dbDir = null;
|
||||
this.dbName = '';
|
||||
this.writeQueue = Promise.resolve();
|
||||
}
|
||||
async open(name) {
|
||||
this.dbName = name;
|
||||
@@ -3021,20 +3024,24 @@
|
||||
async write(key, data) {
|
||||
if (!this.dbDir)
|
||||
return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const fh = await this.dbDir.getFileHandle(key, { create: true });
|
||||
const writable = await fh.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
async delete(key) {
|
||||
if (!this.dbDir)
|
||||
return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
try {
|
||||
await this.dbDir.removeEntry(key);
|
||||
}
|
||||
catch {
|
||||
// 文件不存在则忽略
|
||||
}
|
||||
catch { /* ignore */ }
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
async listKeys() {
|
||||
if (!this.dbDir)
|
||||
@@ -3271,6 +3278,367 @@
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Page Header — 页面头部编解码
|
||||
* @module engine/aria/page/header
|
||||
*/
|
||||
/**
|
||||
* 初始化新页面的 Header。
|
||||
*/
|
||||
function initPageHeader(buf, pageId, type) {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, pageId, false);
|
||||
view.setUint8(4, type);
|
||||
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
|
||||
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
|
||||
view.setUint16(9, 0, false); // slotCount = 0
|
||||
view.setUint32(11, 0, false); // checksum = 0
|
||||
view.setUint8(15, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Page Format — 页面格式整合层
|
||||
* @module engine/aria/page/format
|
||||
*
|
||||
* 组合 Header / Slot / Tuple 操作,提供统一的页面管理接口。
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// 页面创建
|
||||
// ---------------------------------------------------------------------------
|
||||
/** 创建一个新的空页面 */
|
||||
function createPage(pageId, type) {
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, type);
|
||||
return {
|
||||
pageId,
|
||||
type,
|
||||
data,
|
||||
dirty: true,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
|
||||
* @module engine/aria/buffer/eviction
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU 双向链表
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* LRU 链表管理器 — 双向链表,头部是 most recently used,尾部是 least recently used。
|
||||
*/
|
||||
class LRUList {
|
||||
constructor() {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
get size() {
|
||||
return this._size;
|
||||
}
|
||||
/** 将页面移到链表头部。如果是新页面则插入,已存在则移动。 */
|
||||
moveToHead(page) {
|
||||
// 如果已经在头部,无需操作
|
||||
if (this.head === page)
|
||||
return;
|
||||
// 检测是否在链表中
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (inList) {
|
||||
// 先从当前位置移除
|
||||
this.detach(page);
|
||||
}
|
||||
else {
|
||||
this._size++;
|
||||
}
|
||||
// 插入头部
|
||||
page.prev = null;
|
||||
page.next = this.head;
|
||||
if (this.head) {
|
||||
this.head.prev = page;
|
||||
}
|
||||
this.head = page;
|
||||
if (!this.tail) {
|
||||
this.tail = page;
|
||||
}
|
||||
}
|
||||
/** 从链表中移除页面 */
|
||||
remove(page) {
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (!inList)
|
||||
return;
|
||||
this.detach(page);
|
||||
this._size = Math.max(0, this._size - 1);
|
||||
}
|
||||
/** 内部:只调整指针,不修改 _size */
|
||||
detach(page) {
|
||||
if (page.prev) {
|
||||
page.prev.next = page.next;
|
||||
}
|
||||
else if (this.head === page) {
|
||||
this.head = page.next;
|
||||
}
|
||||
if (page.next) {
|
||||
page.next.prev = page.prev;
|
||||
}
|
||||
else if (this.tail === page) {
|
||||
this.tail = page.prev;
|
||||
}
|
||||
page.prev = null;
|
||||
page.next = null;
|
||||
}
|
||||
/** 获取 LRU 尾部(最久未使用的页面) */
|
||||
getLRU() {
|
||||
return this.tail;
|
||||
}
|
||||
/** 弹出 LRU 尾部 */
|
||||
popLRU() {
|
||||
const lru = this.tail;
|
||||
if (lru) {
|
||||
this.remove(lru);
|
||||
}
|
||||
return lru;
|
||||
}
|
||||
/** 清空链表 */
|
||||
clear() {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
/** 获取所有页面(用于迭代) */
|
||||
getAllPages() {
|
||||
const pages = [];
|
||||
let current = this.head;
|
||||
while (current) {
|
||||
pages.push(current);
|
||||
current = current.next;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
|
||||
*/
|
||||
class EvictionManager {
|
||||
constructor(capacity, onEvict) {
|
||||
this.lru = new LRUList();
|
||||
this.capacity = capacity;
|
||||
this.onEvict = onEvict;
|
||||
}
|
||||
/** 访问页面,更新 LRU */
|
||||
access(page) {
|
||||
page.lastAccess = Date.now();
|
||||
this.lru.moveToHead(page);
|
||||
}
|
||||
/** 添加新页面到池中 */
|
||||
add(page) {
|
||||
this.access(page);
|
||||
}
|
||||
/** 移除指定页面 */
|
||||
remove(page) {
|
||||
this.lru.remove(page);
|
||||
}
|
||||
/**
|
||||
* 驱逐页面直到池中有足够空间。
|
||||
* 只驱逐未 pin 的干净页面(dirty=false)。
|
||||
* 如果没有干净页面可驱逐,尝试刷脏页。
|
||||
*/
|
||||
async evictIfNeeded(count) {
|
||||
let evicted = 0;
|
||||
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
|
||||
// 找到可驱逐的页面
|
||||
const victim = this.findEvictionCandidate();
|
||||
if (!victim)
|
||||
break;
|
||||
// 脏页先刷盘
|
||||
if (victim.dirty) {
|
||||
await this.onEvict(victim);
|
||||
victim.dirty = false;
|
||||
}
|
||||
this.lru.remove(victim);
|
||||
evicted++;
|
||||
}
|
||||
return evicted;
|
||||
}
|
||||
/** 查找驱逐候选(优先干净页面,然后最久未用的脏页) */
|
||||
findEvictionCandidate() {
|
||||
// 先从尾部找未 pin 的干净页面
|
||||
let current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0 && !current.dirty)
|
||||
return current;
|
||||
current = current.prev;
|
||||
}
|
||||
// 没有干净页,找未 pin 的脏页
|
||||
current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0)
|
||||
return current;
|
||||
current = current.prev;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/** 获取当前大小 */
|
||||
getSize() {
|
||||
return this.lru.size;
|
||||
}
|
||||
/** 获取容量 */
|
||||
getCapacity() {
|
||||
return this.capacity;
|
||||
}
|
||||
/** 清空 */
|
||||
clear() {
|
||||
this.lru.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine Buffer Pool — 页面缓存池
|
||||
* @module engine/aria/buffer/pool
|
||||
*
|
||||
* 管理固定数量页面的 LRU 缓存,减少对底层储存的访问。
|
||||
*/
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer Pool
|
||||
// ---------------------------------------------------------------------------
|
||||
class BufferPool {
|
||||
constructor(pageIO, capacity = DEFAULT_BUFFER_POOL_PAGES) {
|
||||
this.pages = new Map();
|
||||
this.nextPageId = 0;
|
||||
this.pageIO = pageIO;
|
||||
this.eviction = new EvictionManager(capacity, async (page) => {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
// 页面获取
|
||||
// -----------------------------------------------------------------------
|
||||
/**
|
||||
* 获取页面(必要时从磁盘读取)。
|
||||
* 返回 pin 的页面,使用完成后必须调用 unpin()。
|
||||
*/
|
||||
async getPage(pageId) {
|
||||
// 已在池中
|
||||
let page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.access(page);
|
||||
page.pins++;
|
||||
return page;
|
||||
}
|
||||
// 需要从磁盘加载
|
||||
const buffer = await this.pageIO.readPage(pageId);
|
||||
if (!buffer)
|
||||
return null;
|
||||
// 确保有空间
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
const type = new DataView(buffer).getUint8(4);
|
||||
page = {
|
||||
pageId,
|
||||
type,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 1,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
/**
|
||||
* 创建新页面。
|
||||
*/
|
||||
async newPage(type = PageType.DATA) {
|
||||
const pageId = await this.pageIO.allocatePageId();
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
const page = createPage(pageId, type);
|
||||
page.pins = 1;
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
/**
|
||||
* 释放页面的 pin。
|
||||
*/
|
||||
unpin(page) {
|
||||
if (page.pins > 0) {
|
||||
page.pins--;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 标记页面为脏(需要写回)。
|
||||
*/
|
||||
markDirty(page) {
|
||||
page.dirty = true;
|
||||
}
|
||||
/**
|
||||
* 将脏页面刷新到磁盘。
|
||||
*/
|
||||
async flushPage(pageId) {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page && page.dirty) {
|
||||
await this.pageIO.writePage(pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 刷新所有脏页面。
|
||||
*/
|
||||
async flushAll() {
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 从缓存中删除指定页面(不刷盘)。
|
||||
*/
|
||||
removePage(pageId) {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.remove(page);
|
||||
this.pages.delete(pageId);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 清空缓存池(先刷脏页)。
|
||||
*/
|
||||
async clear() {
|
||||
await this.flushAll();
|
||||
this.pages.clear();
|
||||
this.eviction.clear();
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
// 统计
|
||||
// -----------------------------------------------------------------------
|
||||
/** 获取当前缓存页面数 */
|
||||
getCachedPageCount() {
|
||||
return this.pages.size;
|
||||
}
|
||||
/** 获取缓存容量 */
|
||||
getCapacity() {
|
||||
return this.eviction.getCapacity();
|
||||
}
|
||||
/** 获取脏页面数 */
|
||||
getDirtyPageCount() {
|
||||
let count = 0;
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty)
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AriaEngine — 自研页面式存储引擎主类
|
||||
* @module engine/aria/index
|
||||
@@ -3318,6 +3686,14 @@
|
||||
this.backend = new MemoryBackend();
|
||||
}
|
||||
await this.backend.open(dbName);
|
||||
// 2a. 初始化 Buffer Pool(页面缓存)
|
||||
const pageIO = {
|
||||
readPage: async (pageId) => this.backend.read(`pg_${pageId}`),
|
||||
writePage: async (pageId, data) => this.backend.write(`pg_${pageId}`, data),
|
||||
allocatePageId: async () => Date.now(),
|
||||
freePageId: async () => { },
|
||||
};
|
||||
this.bufferPool = new BufferPool(pageIO, this.config.bufferPoolPages);
|
||||
// 2. 构建 SSTableStore
|
||||
const sstableStore = this.createSSTableStore();
|
||||
// 3. 初始化主 LSM(PK 索引)
|
||||
@@ -4080,6 +4456,84 @@
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
/**
|
||||
* REINDEX: 重建指定表的所有二级索引
|
||||
*/
|
||||
async reindexTable(tableName) {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
const schema = this.schemas.get(tableName);
|
||||
let rebuiltCount = 0;
|
||||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||
if (!colDef.index && !colDef.unique && !colDef.primaryKey)
|
||||
continue;
|
||||
const idxKey = `${tableName}:idx:${colName}`;
|
||||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||
if (!idxLsm)
|
||||
continue;
|
||||
// 清空旧索引
|
||||
await idxLsm.clear();
|
||||
rebuiltCount++;
|
||||
// 从主 LSM 重建索引
|
||||
const rows = this.getAllRows(tableName);
|
||||
for (const row of rows) {
|
||||
const val = row[colName];
|
||||
if (val !== undefined && val !== null) {
|
||||
idxLsm.put(`${String(val)}:${row[this.tablePKs.get(tableName)]}`, { pk: row[this.tablePKs.get(tableName)] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return rebuiltCount;
|
||||
}
|
||||
/**
|
||||
* VACUUM: 压缩 LSM + 清理碎片
|
||||
*/
|
||||
async vacuum() {
|
||||
this.ensureOpen();
|
||||
// 强制 flush memtable
|
||||
await this.lsm.flush();
|
||||
// 压缩各层级
|
||||
for (let level = 0; level < 6; level++) {
|
||||
if (this.lsm.getStats().levelCounts[level] >= 2) {
|
||||
this.lsm.compactLevelSync(level);
|
||||
}
|
||||
}
|
||||
// GC MVCC 版本(保留最新 10 个)
|
||||
const beforeGC = this.mvcc.getActiveTxnCount?.() ?? 0;
|
||||
this.mvcc.gc(10);
|
||||
return { compactedLevels: 6, gcVersions: beforeGC };
|
||||
}
|
||||
/**
|
||||
* 查询优化器:估算各索引成本,选择最优方案
|
||||
*/
|
||||
estimateQueryCost(tableName, query) {
|
||||
const schema = this.schemas.get(tableName);
|
||||
if (!schema || !query.where)
|
||||
return { strategy: 'full_scan', estimatedRows: 0 };
|
||||
const pkCol = this.tablePKs.get(tableName);
|
||||
for (const [col, condition] of Object.entries(query.where)) {
|
||||
if (col === '$and' || col === '$or' || col === '$not')
|
||||
continue;
|
||||
// PK 等值 → 最快,估计 1 行
|
||||
if (col === pkCol && (typeof condition !== 'object' || condition.$eq)) {
|
||||
return { strategy: 'pk_lookup', estimatedRows: 1 };
|
||||
}
|
||||
// 索引列等值 → 快
|
||||
const colDef = schema.columns[col];
|
||||
if (colDef?.index || colDef?.unique) {
|
||||
if (typeof condition !== 'object' || condition.$eq) {
|
||||
return { strategy: `index_eq:${col}`, estimatedRows: 1 };
|
||||
}
|
||||
if (condition.$in && Array.isArray(condition.$in)) {
|
||||
return { strategy: `index_in:${col}`, estimatedRows: condition.$in.length };
|
||||
}
|
||||
if (condition.$gt || condition.$lt || condition.$gte || condition.$lte) {
|
||||
return { strategy: `index_range:${col}`, estimatedRows: 100 };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { strategy: 'full_scan', estimatedRows: 1000 };
|
||||
}
|
||||
ensureOpen() {
|
||||
if (!this.opened)
|
||||
throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* AriaEngine Performance Benchmark
|
||||
* 测量 insert/find/update/delete 在不同数据量级下的性能
|
||||
*/
|
||||
import { AriaEngine } from './index';
|
||||
import { createSchema } from '../../table/schema';
|
||||
|
||||
interface BenchResult {
|
||||
name: string;
|
||||
rows: number;
|
||||
ops: number;
|
||||
totalMs: number;
|
||||
opsPerSec: number;
|
||||
}
|
||||
|
||||
function createTestSchema() {
|
||||
return createSchema('bench', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string' },
|
||||
value: { type: 'number', index: true },
|
||||
});
|
||||
}
|
||||
|
||||
async function timeIt(name: string, fn: () => Promise<void>): Promise<number> {
|
||||
const start = performance.now();
|
||||
await fn();
|
||||
return performance.now() - start;
|
||||
}
|
||||
|
||||
export async function runBenchmarks(): Promise<BenchResult[]> {
|
||||
const results: BenchResult[] = [];
|
||||
const sizes = [100, 1000, 10000, 100000];
|
||||
|
||||
for (const size of sizes) {
|
||||
const engine = new AriaEngine({ storageBackend: 'memory' });
|
||||
await engine.open('bench-db', 1);
|
||||
await engine.createTable(createTestSchema());
|
||||
|
||||
// Insert
|
||||
const rows = [];
|
||||
for (let i = 0; i < size; i++) {
|
||||
rows.push({ id: `${i}`, name: `User${i}`, value: i * 10 });
|
||||
}
|
||||
const insertMs = await timeIt(`insert-${size}`, () => engine.insert('bench', rows));
|
||||
results.push({ name: `insert-${size}`, rows: size, ops: size, totalMs: insertMs, opsPerSec: Math.round(size / (insertMs / 1000)) });
|
||||
|
||||
// Find by PK
|
||||
const findMs = await timeIt(`find-pk-${size}`, () => engine.find('bench', { table: 'bench', where: { id: `${size - 1}` } }));
|
||||
results.push({ name: `find-pk-${size}`, rows: size, ops: 1, totalMs: findMs, opsPerSec: Math.round(1000 / findMs) });
|
||||
|
||||
// Find by index
|
||||
const findIdxMs = await timeIt(`find-idx-${size}`, () => engine.find('bench', { table: 'bench', where: { value: (size - 1) * 10 } }));
|
||||
results.push({ name: `find-idx-${size}`, rows: size, ops: 1, totalMs: findIdxMs, opsPerSec: Math.round(1000 / findIdxMs) });
|
||||
|
||||
// Update
|
||||
const updateMs = await timeIt(`update-${size}`, () => engine.update('bench', { table: 'bench', where: { id: `${size - 1}` } }, { name: 'Updated' }));
|
||||
results.push({ name: `update-${size}`, rows: size, ops: 1, totalMs: updateMs, opsPerSec: Math.round(1000 / updateMs) });
|
||||
|
||||
// Delete
|
||||
const deleteMs = await timeIt(`delete-${size}`, () => engine.delete('bench', { table: 'bench', where: { id: `${size - 1}` } }));
|
||||
results.push({ name: `delete-${size}`, rows: size, ops: 1, totalMs: deleteMs, opsPerSec: Math.round(1000 / deleteMs) });
|
||||
|
||||
await engine.close();
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { IndexedDBBackend, MemoryBackend, type IStorageBackend } from './store/b
|
||||
import { OPFSBackend } from './store/opfs_backend';
|
||||
import { MVCCManager } from './transaction/mvcc';
|
||||
import { BloomFilter } from './index/bloom';
|
||||
import { BufferPool, type PageIO } from './buffer/pool';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// AriaEngine
|
||||
@@ -52,6 +53,9 @@ export class AriaEngine implements IStorageEngine {
|
||||
private txnSnapshot: Map<string, Record<string, unknown>> | null = null;
|
||||
private gcCounter = 0;
|
||||
|
||||
// Buffer Pool(页面缓存,减少磁盘 I/O)
|
||||
private bufferPool!: BufferPool;
|
||||
|
||||
constructor(config: AriaEngineConfig = {}) {
|
||||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||||
}
|
||||
@@ -74,6 +78,15 @@ export class AriaEngine implements IStorageEngine {
|
||||
}
|
||||
await this.backend.open(dbName);
|
||||
|
||||
// 2a. 初始化 Buffer Pool(页面缓存)
|
||||
const pageIO: PageIO = {
|
||||
readPage: async (pageId) => this.backend.read(`pg_${pageId}`),
|
||||
writePage: async (pageId, data) => this.backend.write(`pg_${pageId}`, data),
|
||||
allocatePageId: async () => Date.now(),
|
||||
freePageId: async () => {},
|
||||
};
|
||||
this.bufferPool = new BufferPool(pageIO, this.config.bufferPoolPages);
|
||||
|
||||
// 2. 构建 SSTableStore
|
||||
const sstableStore = this.createSSTableStore();
|
||||
|
||||
@@ -909,6 +922,90 @@ export class AriaEngine implements IStorageEngine {
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* REINDEX: 重建指定表的所有二级索引
|
||||
*/
|
||||
async reindexTable(tableName: string): Promise<number> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
const schema = this.schemas.get(tableName)!;
|
||||
let rebuiltCount = 0;
|
||||
|
||||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||
if (!colDef.index && !colDef.unique && !colDef.primaryKey) continue;
|
||||
const idxKey = `${tableName}:idx:${colName}`;
|
||||
const idxLsm = this.secondaryIndexes.get(idxKey);
|
||||
if (!idxLsm) continue;
|
||||
|
||||
// 清空旧索引
|
||||
await idxLsm.clear();
|
||||
rebuiltCount++;
|
||||
|
||||
// 从主 LSM 重建索引
|
||||
const rows = this.getAllRows(tableName);
|
||||
for (const row of rows) {
|
||||
const val = row[colName];
|
||||
if (val !== undefined && val !== null) {
|
||||
idxLsm.put(`${String(val)}:${row[this.tablePKs.get(tableName)!]}`, { pk: row[this.tablePKs.get(tableName)!] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return rebuiltCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* VACUUM: 压缩 LSM + 清理碎片
|
||||
*/
|
||||
async vacuum(): Promise<{ compactedLevels: number; gcVersions: number }> {
|
||||
this.ensureOpen();
|
||||
// 强制 flush memtable
|
||||
await this.lsm.flush();
|
||||
// 压缩各层级
|
||||
for (let level = 0; level < 6; level++) {
|
||||
if (this.lsm.getStats().levelCounts[level] >= 2) {
|
||||
(this.lsm as any).compactLevelSync(level);
|
||||
}
|
||||
}
|
||||
// GC MVCC 版本(保留最新 10 个)
|
||||
const beforeGC = this.mvcc.getActiveTxnCount?.() ?? 0;
|
||||
this.mvcc.gc(10);
|
||||
return { compactedLevels: 6, gcVersions: beforeGC };
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询优化器:估算各索引成本,选择最优方案
|
||||
*/
|
||||
estimateQueryCost(tableName: string, query: QueryPlan): { strategy: string; estimatedRows: number } {
|
||||
const schema = this.schemas.get(tableName);
|
||||
if (!schema || !query.where) return { strategy: 'full_scan', estimatedRows: 0 };
|
||||
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
for (const [col, condition] of Object.entries(query.where)) {
|
||||
if (col === '$and' || col === '$or' || col === '$not') continue;
|
||||
|
||||
// PK 等值 → 最快,估计 1 行
|
||||
if (col === pkCol && (typeof condition !== 'object' || (condition as any).$eq)) {
|
||||
return { strategy: 'pk_lookup', estimatedRows: 1 };
|
||||
}
|
||||
|
||||
// 索引列等值 → 快
|
||||
const colDef = schema.columns[col];
|
||||
if (colDef?.index || colDef?.unique) {
|
||||
if (typeof condition !== 'object' || (condition as any).$eq) {
|
||||
return { strategy: `index_eq:${col}`, estimatedRows: 1 };
|
||||
}
|
||||
if ((condition as any).$in && Array.isArray((condition as any).$in)) {
|
||||
return { strategy: `index_in:${col}`, estimatedRows: (condition as any).$in.length };
|
||||
}
|
||||
if ((condition as any).$gt || (condition as any).$lt || (condition as any).$gte || (condition as any).$lte) {
|
||||
return { strategy: `index_range:${col}`, estimatedRows: 100 };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { strategy: 'full_scan', estimatedRows: 1000 };
|
||||
}
|
||||
|
||||
private ensureOpen(): void {
|
||||
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ export class OPFSBackend implements IStorageBackend {
|
||||
private root: FileSystemDirectoryHandle | null = null;
|
||||
private dbDir: FileSystemDirectoryHandle | null = null;
|
||||
private dbName = '';
|
||||
private writeQueue: Promise<void> = Promise.resolve();
|
||||
|
||||
async open(name: string): Promise<void> {
|
||||
this.dbName = name;
|
||||
@@ -42,19 +43,21 @@ export class OPFSBackend implements IStorageBackend {
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
const fh = await this.dbDir.getFileHandle(key, { create: true });
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const fh = await this.dbDir!.getFileHandle(key, { create: true });
|
||||
const writable = await fh.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
try {
|
||||
await this.dbDir.removeEntry(key);
|
||||
} catch {
|
||||
// 文件不存在则忽略
|
||||
}
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
try { await this.dbDir!.removeEntry(key); } catch { /* ignore */ }
|
||||
});
|
||||
return this.writeQueue;
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
|
||||
Reference in New Issue
Block a user