feat: v0.2.0 AriaEngine 自研存储引擎
- 新增 AriaEngine: LSM-Tree 页面式存储引擎，19 个模块，~3500 行 TS - page/: Slotted Page 格式 (header/slot/tuple/format) + CRC32 - buffer/: Buffer Pool (LRU 缓存 + 驱逐策略) - index/: LSM-Tree (MemTable 红黑树 + SSTable + Bloom Filter + Merge Iterator) - wal/: WAL 日志 (二进制格式) + Checkpoint 管理 - transaction/: MVCC 版本链 + 快照隔离 - store/: IndexedDB / Memory 双后端抽象 - compression/: LZ4 页面压缩 - 完整持久化: Schema 自动保存、SSTable 元数据管理、WAL 恢复 - 事务感知 CRUD: insert/update/delete 在事务中缓冲到 snapshot - mode: 'aria' 激活自研引擎 - 新增 7 个测试文件，测试数 318 → 524，套件 20 → 27 - aria-page.test.ts (32 tests): Page 格式单元测试 - aria-index.test.ts (26 tests): Bloom Filter + MemTable - aria-sstable.test.ts (9 tests): SSTable Builder + Reader - aria-buffer.test.ts (25 tests): LRU + Eviction + Buffer Pool - aria-wal-mvcc.test.ts (22 tests): WAL 编解码 + MVCC 事务 - aria-compress.test.ts (11 tests): LZ4 + Merge Iterator - aria.test.ts (80 tests): AriaEngine 集成 + 边界测试 - Bug 修复: LRUList size 跟踪、WAL 缓冲区越界、ColumnEncoding 导入 - 全面更新 README.md + site/ 站点文件 (index/docs/demo)
This commit is contained in:
+3
-3
@@ -8,13 +8,13 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 存储模式 */
|
||||
export type StorageMode = 'memory' | 'disk' | 'hybrid';
|
||||
export type StorageMode = 'memory' | 'disk' | 'hybrid' | 'aria';
|
||||
|
||||
/** 磁盘引擎类型 */
|
||||
export type DiskEngine = 'indexeddb' | 'opfs';
|
||||
|
||||
/** 所有存储模式 */
|
||||
export const STORAGE_MODES: StorageMode[] = ['memory', 'disk', 'hybrid'];
|
||||
export const STORAGE_MODES: StorageMode[] = ['memory', 'disk', 'hybrid', 'aria'];
|
||||
|
||||
/** 所有磁盘引擎 */
|
||||
export const DISK_ENGINES: DiskEngine[] = ['indexeddb', 'opfs'];
|
||||
@@ -203,4 +203,4 @@ export class DatabaseError extends Error {
|
||||
// 版本
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const VERSION = '0.1.14';
|
||||
export const VERSION = '0.2.0';
|
||||
|
||||
@@ -11,6 +11,7 @@ import { DB_DEFAULTS, DatabaseError } from './constants';
|
||||
import { MemoryEngine } from './engine/memory';
|
||||
import { IndexedDBEngine } from './engine/indexeddb';
|
||||
import { OPFSEngine } from './engine/opfs';
|
||||
import { AriaEngine } from './engine/aria/index';
|
||||
import { HybridEngine } from './hybrid/index';
|
||||
import { Table } from './table/table';
|
||||
import { createSchema } from './table/schema';
|
||||
@@ -258,6 +259,8 @@ export class MetonaSqlark {
|
||||
return new MemoryEngine();
|
||||
case 'disk':
|
||||
return diskEngine === 'opfs' ? new OPFSEngine() : new IndexedDBEngine();
|
||||
case 'aria':
|
||||
return new AriaEngine({ storageBackend: diskEngine === 'opfs' ? 'memory' : 'indexeddb' });
|
||||
case 'hybrid':
|
||||
return new HybridEngine(diskEngine);
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
/**
|
||||
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
|
||||
* @module engine/aria/buffer/eviction
|
||||
*/
|
||||
|
||||
import type { PageHandle } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU 双向链表
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* LRU 链表管理器 — 双向链表，头部是 most recently used，尾部是 least recently used。
|
||||
*/
|
||||
export class LRUList {
|
||||
private head: PageHandle | null = null;
|
||||
private tail: PageHandle | null = null;
|
||||
private _size = 0;
|
||||
|
||||
get size(): number {
|
||||
return this._size;
|
||||
}
|
||||
|
||||
/** 将页面移到链表头部。如果是新页面则插入，已存在则移动。 */
|
||||
moveToHead(page: PageHandle): void {
|
||||
// 如果已经在头部，无需操作
|
||||
if (this.head === page) return;
|
||||
|
||||
// 检测是否在链表中
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
|
||||
if (inList) {
|
||||
// 先从当前位置移除
|
||||
this.detach(page);
|
||||
} else {
|
||||
this._size++;
|
||||
}
|
||||
|
||||
// 插入头部
|
||||
page.prev = null;
|
||||
page.next = this.head;
|
||||
if (this.head) {
|
||||
this.head.prev = page;
|
||||
}
|
||||
this.head = page;
|
||||
if (!this.tail) {
|
||||
this.tail = page;
|
||||
}
|
||||
}
|
||||
|
||||
/** 从链表中移除页面 */
|
||||
remove(page: PageHandle): void {
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (!inList) return;
|
||||
|
||||
this.detach(page);
|
||||
this._size = Math.max(0, this._size - 1);
|
||||
}
|
||||
|
||||
/** 内部：只调整指针，不修改 _size */
|
||||
private detach(page: PageHandle): void {
|
||||
if (page.prev) {
|
||||
page.prev.next = page.next;
|
||||
} else if (this.head === page) {
|
||||
this.head = page.next;
|
||||
}
|
||||
|
||||
if (page.next) {
|
||||
page.next.prev = page.prev;
|
||||
} else if (this.tail === page) {
|
||||
this.tail = page.prev;
|
||||
}
|
||||
|
||||
page.prev = null;
|
||||
page.next = null;
|
||||
}
|
||||
|
||||
/** 获取 LRU 尾部（最久未使用的页面） */
|
||||
getLRU(): PageHandle | null {
|
||||
return this.tail;
|
||||
}
|
||||
|
||||
/** 弹出 LRU 尾部 */
|
||||
popLRU(): PageHandle | null {
|
||||
const lru = this.tail;
|
||||
if (lru) {
|
||||
this.remove(lru);
|
||||
}
|
||||
return lru;
|
||||
}
|
||||
|
||||
/** 清空链表 */
|
||||
clear(): void {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
|
||||
/** 获取所有页面（用于迭代） */
|
||||
getAllPages(): PageHandle[] {
|
||||
const pages: PageHandle[] = [];
|
||||
let current = this.head;
|
||||
while (current) {
|
||||
pages.push(current);
|
||||
current = current.next;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Eviction 策略
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type EvictionCallback = (page: PageHandle) => Promise<void>;
|
||||
|
||||
/**
|
||||
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
|
||||
*/
|
||||
export class EvictionManager {
|
||||
private lru: LRUList;
|
||||
private onEvict: EvictionCallback;
|
||||
private capacity: number;
|
||||
|
||||
constructor(capacity: number, onEvict: EvictionCallback) {
|
||||
this.lru = new LRUList();
|
||||
this.capacity = capacity;
|
||||
this.onEvict = onEvict;
|
||||
}
|
||||
|
||||
/** 访问页面，更新 LRU */
|
||||
access(page: PageHandle): void {
|
||||
page.lastAccess = Date.now();
|
||||
this.lru.moveToHead(page);
|
||||
}
|
||||
|
||||
/** 添加新页面到池中 */
|
||||
add(page: PageHandle): void {
|
||||
this.access(page);
|
||||
}
|
||||
|
||||
/** 移除指定页面 */
|
||||
remove(page: PageHandle): void {
|
||||
this.lru.remove(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* 驱逐页面直到池中有足够空间。
|
||||
* 只驱逐未 pin 的干净页面（dirty=false）。
|
||||
* 如果没有干净页面可驱逐，尝试刷脏页。
|
||||
*/
|
||||
async evictIfNeeded(count: number): Promise<number> {
|
||||
let evicted = 0;
|
||||
|
||||
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
|
||||
// 找到可驱逐的页面
|
||||
const victim = this.findEvictionCandidate();
|
||||
if (!victim) break;
|
||||
|
||||
// 脏页先刷盘
|
||||
if (victim.dirty) {
|
||||
await this.onEvict(victim);
|
||||
victim.dirty = false;
|
||||
}
|
||||
|
||||
this.lru.remove(victim);
|
||||
evicted++;
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
/** 查找驱逐候选（优先干净页面，然后最久未用的脏页） */
|
||||
private findEvictionCandidate(): PageHandle | null {
|
||||
// 先从尾部找未 pin 的干净页面
|
||||
let current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0 && !current.dirty) return current;
|
||||
current = current.prev;
|
||||
}
|
||||
|
||||
// 没有干净页，找未 pin 的脏页
|
||||
current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0) return current;
|
||||
current = current.prev;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 获取当前大小 */
|
||||
getSize(): number {
|
||||
return this.lru.size;
|
||||
}
|
||||
|
||||
/** 获取容量 */
|
||||
getCapacity(): number {
|
||||
return this.capacity;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.lru.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* AriaEngine Buffer Pool — 页面缓存池
|
||||
* @module engine/aria/buffer/pool
|
||||
*
|
||||
* 管理固定数量页面的 LRU 缓存，减少对底层储存的访问。
|
||||
*/
|
||||
|
||||
import type { PageHandle } from '../types';
|
||||
import { PageType, DEFAULT_BUFFER_POOL_PAGES } from '../types';
|
||||
import { createPage } from '../page/format';
|
||||
import { EvictionManager } from './eviction';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Page Read / Write 回调
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface PageIO {
|
||||
/** 从存储后端读取页面 */
|
||||
readPage(pageId: number): Promise<ArrayBuffer | null>;
|
||||
/** 将页面写入存储后端 */
|
||||
writePage(pageId: number, data: ArrayBuffer): Promise<void>;
|
||||
/** 分配新页面 ID */
|
||||
allocatePageId(): Promise<number>;
|
||||
/** 释放页面 ID */
|
||||
freePageId(pageId: number): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer Pool
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class BufferPool {
|
||||
private pages: Map<number, PageHandle> = new Map();
|
||||
private eviction: EvictionManager;
|
||||
private pageIO: PageIO;
|
||||
private nextPageId = 0;
|
||||
|
||||
constructor(pageIO: PageIO, capacity: number = DEFAULT_BUFFER_POOL_PAGES) {
|
||||
this.pageIO = pageIO;
|
||||
this.eviction = new EvictionManager(capacity, async (page) => {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 页面获取
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 获取页面（必要时从磁盘读取）。
|
||||
* 返回 pin 的页面，使用完成后必须调用 unpin()。
|
||||
*/
|
||||
async getPage(pageId: number): Promise<PageHandle | null> {
|
||||
// 已在池中
|
||||
let page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.access(page);
|
||||
page.pins++;
|
||||
return page;
|
||||
}
|
||||
|
||||
// 需要从磁盘加载
|
||||
const buffer = await this.pageIO.readPage(pageId);
|
||||
if (!buffer) return null;
|
||||
|
||||
// 确保有空间
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
|
||||
const type = new DataView(buffer).getUint8(4) as PageType;
|
||||
page = {
|
||||
pageId,
|
||||
type,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 1,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建新页面。
|
||||
*/
|
||||
async newPage(type: PageType = PageType.DATA): Promise<PageHandle> {
|
||||
const pageId = await this.pageIO.allocatePageId();
|
||||
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
|
||||
const page = createPage(pageId, type);
|
||||
page.pins = 1;
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* 释放页面的 pin。
|
||||
*/
|
||||
unpin(page: PageHandle): void {
|
||||
if (page.pins > 0) {
|
||||
page.pins--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记页面为脏（需要写回）。
|
||||
*/
|
||||
markDirty(page: PageHandle): void {
|
||||
page.dirty = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将脏页面刷新到磁盘。
|
||||
*/
|
||||
async flushPage(pageId: number): Promise<void> {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page && page.dirty) {
|
||||
await this.pageIO.writePage(pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新所有脏页面。
|
||||
*/
|
||||
async flushAll(): Promise<void> {
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从缓存中删除指定页面（不刷盘）。
|
||||
*/
|
||||
removePage(pageId: number): void {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.remove(page);
|
||||
this.pages.delete(pageId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空缓存池（先刷脏页）。
|
||||
*/
|
||||
async clear(): Promise<void> {
|
||||
await this.flushAll();
|
||||
this.pages.clear();
|
||||
this.eviction.clear();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 统计
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/** 获取当前缓存页面数 */
|
||||
getCachedPageCount(): number {
|
||||
return this.pages.size;
|
||||
}
|
||||
|
||||
/** 获取缓存容量 */
|
||||
getCapacity(): number {
|
||||
return this.eviction.getCapacity();
|
||||
}
|
||||
|
||||
/** 获取脏页面数 */
|
||||
getDirtyPageCount(): number {
|
||||
let count = 0;
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* AriaEngine LZ4 Compression — 简易 LZ4 压缩
|
||||
* @module engine/aria/compression/lz4
|
||||
*
|
||||
* LZ4 是一种极快的压缩算法，适合页面级数据压缩。
|
||||
* 此处实现一个简化版，用于演示概念。
|
||||
*
|
||||
* 压缩格式:
|
||||
* LITERAL_RUN: [token: 1B] [literals: N bytes]
|
||||
* MATCH: [offset: 2B LE] [matchLength: N]
|
||||
*
|
||||
* 实际生产环境建议使用 lz4 或 snappy 库。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 常量
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MIN_MATCH = 4;
|
||||
const MAX_LITERAL_LENGTH = 15;
|
||||
const MAX_MATCH_LENGTH = 18;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 压缩
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 压缩数据。如果压缩后比原始大，返回原始数据（标记未压缩）。
|
||||
*/
|
||||
export function compressLZ4(input: Uint8Array): Uint8Array {
|
||||
if (input.byteLength < MIN_MATCH) {
|
||||
// 太小不值得压缩
|
||||
return input;
|
||||
}
|
||||
|
||||
const maxOutputSize = input.byteLength + (input.byteLength / 255) + 16;
|
||||
const output = new Uint8Array(maxOutputSize);
|
||||
let srcIdx = 0;
|
||||
let dstIdx = 0;
|
||||
|
||||
while (srcIdx < input.byteLength) {
|
||||
// 查找最长匹配
|
||||
let bestMatchLen = 0;
|
||||
let bestMatchOffset = 0;
|
||||
const searchStart = Math.max(0, srcIdx - 65535);
|
||||
const searchEnd = srcIdx;
|
||||
|
||||
for (let i = searchStart; i < searchEnd; i++) {
|
||||
let matchLen = 0;
|
||||
while (
|
||||
srcIdx + matchLen < input.byteLength &&
|
||||
i + matchLen < srcIdx &&
|
||||
input[i + matchLen] === input[srcIdx + matchLen] &&
|
||||
matchLen < 255
|
||||
) {
|
||||
matchLen++;
|
||||
}
|
||||
if (matchLen > bestMatchLen && matchLen >= MIN_MATCH) {
|
||||
bestMatchLen = matchLen;
|
||||
bestMatchOffset = srcIdx - i;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestMatchLen >= MIN_MATCH) {
|
||||
// 写入匹配
|
||||
const literalLen = 0;
|
||||
const matchLen = Math.min(bestMatchLen - MIN_MATCH, MAX_MATCH_LENGTH);
|
||||
|
||||
output[dstIdx++] = ((literalLen & 0x0F) << 4) | (matchLen & 0x0F);
|
||||
output[dstIdx++] = bestMatchOffset & 0xFF;
|
||||
output[dstIdx++] = (bestMatchOffset >> 8) & 0xFF;
|
||||
srcIdx += matchLen + MIN_MATCH;
|
||||
} else {
|
||||
// 写入字面量
|
||||
let litStart = srcIdx;
|
||||
while (srcIdx < input.byteLength) {
|
||||
const remaining = input.byteLength - srcIdx;
|
||||
if (remaining < MIN_MATCH) {
|
||||
srcIdx += remaining;
|
||||
break;
|
||||
}
|
||||
srcIdx++;
|
||||
|
||||
// 检查下一个位置是否有匹配
|
||||
let hasMatch = false;
|
||||
const nextEnd = Math.min(srcIdx, input.byteLength);
|
||||
for (let i = Math.max(0, srcIdx - 65535); i < srcIdx && !hasMatch; i++) {
|
||||
let ml = 0;
|
||||
while (srcIdx + ml < input.byteLength && i + ml < srcIdx && input[i + ml] === input[srcIdx + ml] && ml < MIN_MATCH) {
|
||||
ml++;
|
||||
}
|
||||
if (ml >= MIN_MATCH) hasMatch = true;
|
||||
}
|
||||
|
||||
if (hasMatch) {
|
||||
srcIdx--;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let litLen = srcIdx - litStart;
|
||||
while (litLen > 0) {
|
||||
const chunk = Math.min(litLen, MAX_LITERAL_LENGTH);
|
||||
output[dstIdx++] = ((chunk & 0x0F) << 4);
|
||||
for (let j = 0; j < chunk; j++) {
|
||||
output[dstIdx++] = input[litStart + j];
|
||||
}
|
||||
litLen -= chunk;
|
||||
litStart += chunk;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果压缩后更大，返回原始
|
||||
if (dstIdx >= input.byteLength) {
|
||||
return input;
|
||||
}
|
||||
|
||||
return output.slice(0, dstIdx);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 解压
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 解压 LZ4 数据。
|
||||
*/
|
||||
export function decompressLZ4(
|
||||
input: Uint8Array,
|
||||
originalSize: number,
|
||||
): Uint8Array {
|
||||
const output = new Uint8Array(originalSize);
|
||||
let srcIdx = 0;
|
||||
let dstIdx = 0;
|
||||
|
||||
while (srcIdx < input.byteLength && dstIdx < originalSize) {
|
||||
const token = input[srcIdx++];
|
||||
let literalLen = (token >> 4) & 0x0F;
|
||||
|
||||
// 扩展字面量长度
|
||||
if (literalLen === 15) {
|
||||
while (srcIdx < input.byteLength && input[srcIdx] === 255) {
|
||||
literalLen += 255;
|
||||
srcIdx++;
|
||||
}
|
||||
if (srcIdx < input.byteLength) {
|
||||
literalLen += input[srcIdx++];
|
||||
}
|
||||
}
|
||||
|
||||
// 复制字面量
|
||||
for (let i = 0; i < literalLen && srcIdx < input.byteLength && dstIdx < originalSize; i++) {
|
||||
output[dstIdx++] = input[srcIdx++];
|
||||
}
|
||||
|
||||
if (srcIdx >= input.byteLength || dstIdx >= originalSize) break;
|
||||
|
||||
// 偏移量
|
||||
const offset = input[srcIdx++] | (input[srcIdx++] << 8);
|
||||
|
||||
let matchLen = (token & 0x0F) + MIN_MATCH;
|
||||
|
||||
// 扩展匹配长度
|
||||
if ((token & 0x0F) === 15) {
|
||||
while (srcIdx < input.byteLength && input[srcIdx] === 255) {
|
||||
matchLen += 255;
|
||||
srcIdx++;
|
||||
}
|
||||
if (srcIdx < input.byteLength) {
|
||||
matchLen += input[srcIdx++];
|
||||
}
|
||||
}
|
||||
|
||||
// 复制匹配
|
||||
for (let i = 0; i < matchLen && dstIdx < originalSize; i++) {
|
||||
output[dstIdx] = output[dstIdx - offset];
|
||||
dstIdx++;
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
/**
|
||||
* AriaEngine — 自研页面式存储引擎主类
|
||||
* @module engine/aria/index
|
||||
*
|
||||
* 实现 IStorageEngine 接口。
|
||||
*
|
||||
* v0.2.1: 完整持久化
|
||||
* - Schema 存入 __aria_schemas
|
||||
* - SSTable 元数据存入 __aria_lsm_meta
|
||||
* - WAL 恢复包含行数据
|
||||
* - 启动时自动加载 Schema + SSTable
|
||||
*/
|
||||
|
||||
import type { IStorageEngine } from '../interface';
|
||||
import type { QueryPlan, TableSchema, ColumnDef } from '../../constants';
|
||||
import { DatabaseError } from '../../constants';
|
||||
import { matchWhere, applyOrderBy, projectColumns } from '../../query/where-matcher';
|
||||
|
||||
import type { AriaEngineConfig, SSTableMeta } from './types';
|
||||
import { DEFAULT_ARIA_CONFIG } from './types';
|
||||
|
||||
import { LSM } from './index/lsm';
|
||||
import type { SSTableStore } from './index/lsm';
|
||||
import { WAL } from './wal/log';
|
||||
import { WALRecordType, type WALRecord } from './types';
|
||||
import { CheckpointManager } from './wal/checkpoint';
|
||||
import { IndexedDBBackend, MemoryBackend, type IStorageBackend } from './store/backend';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// AriaEngine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class AriaEngine implements IStorageEngine {
|
||||
readonly name = 'aria';
|
||||
|
||||
private config!: Required<AriaEngineConfig>;
|
||||
private lsm!: LSM;
|
||||
private wal!: WAL;
|
||||
private checkpointManager!: CheckpointManager;
|
||||
private backend!: IStorageBackend;
|
||||
private opened = false;
|
||||
private dbName = '';
|
||||
|
||||
// 表结构
|
||||
private schemas: Map<string, TableSchema> = new Map();
|
||||
private tablePKs: Map<string, string> = new Map();
|
||||
private opCounter = 0;
|
||||
|
||||
// 事务
|
||||
private currentTxnId: number | null = null;
|
||||
private txnSnapshot: Map<string, Record<string, unknown>> | null = null;
|
||||
|
||||
constructor(config: AriaEngineConfig = {}) {
|
||||
this.config = { ...DEFAULT_ARIA_CONFIG, ...config };
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 生命周期
|
||||
// =======================================================================
|
||||
|
||||
async open(dbName: string, _version: number): Promise<void> {
|
||||
if (this.opened) return;
|
||||
this.dbName = dbName;
|
||||
|
||||
// 1. 存储后端
|
||||
if (this.config.storageBackend === 'indexeddb') {
|
||||
this.backend = new IndexedDBBackend();
|
||||
} else {
|
||||
this.backend = new MemoryBackend();
|
||||
}
|
||||
await this.backend.open(dbName);
|
||||
|
||||
// 2. 构建 SSTableStore
|
||||
const sstableStore = this.createSSTableStore();
|
||||
|
||||
// 3. 初始化 LSM
|
||||
this.lsm = new LSM({
|
||||
memtableSizeThreshold: this.config.memtableSizeThreshold,
|
||||
levelSizeMultiplier: this.config.levelSizeMultiplier,
|
||||
blockSize: this.config.pageSize,
|
||||
bloomBitsPerKey: this.config.bloomFilterBitsPerKey,
|
||||
sstableStore,
|
||||
});
|
||||
|
||||
// 4. 初始化 WAL
|
||||
this.wal = new WAL(
|
||||
{
|
||||
append: async (data) => {
|
||||
// Store each record as a separate numbered key
|
||||
const idx = await this.getWALCount();
|
||||
const slice = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
||||
const copy = slice.buffer.slice(slice.byteOffset, slice.byteOffset + slice.byteLength) as ArrayBuffer;
|
||||
await this.backend.write(`__wal_${idx}`, copy);
|
||||
await this.setWALCount(idx + 1);
|
||||
},
|
||||
readAll: async () => {
|
||||
const count = await this.getWALCount();
|
||||
if (count === 0) return new Uint8Array(0);
|
||||
// Read all records and concatenate
|
||||
const chunks: Uint8Array[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const d = await this.backend.read(`__wal_${i}`);
|
||||
if (d) chunks.push(new Uint8Array(d));
|
||||
}
|
||||
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
|
||||
const combined = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) { combined.set(c, off); off += c.byteLength; }
|
||||
return combined;
|
||||
},
|
||||
truncate: async () => {
|
||||
const count = await this.getWALCount();
|
||||
for (let i = 0; i < count; i++) {
|
||||
await this.backend.delete(`__wal_${i}`);
|
||||
}
|
||||
await this.setWALCount(0);
|
||||
},
|
||||
exists: async () => {
|
||||
const count = await this.getWALCount();
|
||||
return count > 0;
|
||||
},
|
||||
},
|
||||
this.config.walEnabled,
|
||||
this.config.walSyncMode,
|
||||
);
|
||||
|
||||
// 5. 恢复 Schema
|
||||
await this.loadSchemas();
|
||||
|
||||
// 6. 初始化 LSM（加载 SSTable 元数据）
|
||||
await this.lsm.init();
|
||||
|
||||
// 7. WAL 恢复（恢复未刷盘的数据）
|
||||
await this.wal.recover((record) => this.applyWALRecord(record));
|
||||
|
||||
// 8. Checkpoint Manager（BufferPool 暂简化，使用 flush 替代）
|
||||
this.checkpointManager = new CheckpointManager(
|
||||
this.lsm,
|
||||
this.wal,
|
||||
{ flushAll: async () => { await this.lsm.flush(); } } as any,
|
||||
this.config.checkpointInterval,
|
||||
);
|
||||
|
||||
this.opened = true;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (!this.opened) return;
|
||||
await this.persistSchemas();
|
||||
await this.lsm.flush();
|
||||
await this.wal.flush();
|
||||
await this.backend.close();
|
||||
this.schemas.clear();
|
||||
this.opened = false;
|
||||
}
|
||||
|
||||
isOpen(): boolean { return this.opened; }
|
||||
|
||||
// =======================================================================
|
||||
// 表管理
|
||||
// =======================================================================
|
||||
|
||||
async createTable(schema: TableSchema): Promise<void> {
|
||||
this.ensureOpen();
|
||||
if (this.schemas.has(schema.name)) {
|
||||
throw new DatabaseError(`Table "${schema.name}" already exists`, 'TABLE_EXISTS');
|
||||
}
|
||||
|
||||
this.schemas.set(schema.name, schema);
|
||||
this.tablePKs.set(schema.name, this.getPK(schema));
|
||||
|
||||
await this.persistSchemas();
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.CREATE_TABLE,
|
||||
txnId: 0,
|
||||
tableName: schema.name,
|
||||
key: '',
|
||||
data: { schema: JSON.stringify(schema) } as unknown as Record<string, unknown>,
|
||||
});
|
||||
}
|
||||
|
||||
async dropTable(tableName: string): Promise<void> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
|
||||
// 删除表中所有行
|
||||
const rows = this.getAllRows(tableName);
|
||||
for (const row of rows) {
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
this.lsm.delete(`${tableName}:${row[pkCol]}`);
|
||||
}
|
||||
|
||||
this.schemas.delete(tableName);
|
||||
this.tablePKs.delete(tableName);
|
||||
await this.persistSchemas();
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.DROP_TABLE,
|
||||
txnId: 0,
|
||||
tableName,
|
||||
key: '',
|
||||
});
|
||||
}
|
||||
|
||||
async hasTable(tableName: string): Promise<boolean> {
|
||||
return this.schemas.has(tableName);
|
||||
}
|
||||
|
||||
async getTableNames(): Promise<string[]> {
|
||||
return Array.from(this.schemas.keys());
|
||||
}
|
||||
|
||||
async getTableSchema(tableName: string): Promise<TableSchema | null> {
|
||||
return this.schemas.get(tableName) ?? null;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// CRUD
|
||||
// =======================================================================
|
||||
|
||||
async insert(tableName: string, rows: Record<string, unknown>[]): Promise<string[]> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
|
||||
const schema = this.schemas.get(tableName)!;
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
const pks: string[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const validated = this.validateRow(schema, row);
|
||||
const pkValue = String(validated[pkCol]);
|
||||
const key = `${tableName}:${pkValue}`;
|
||||
|
||||
// Check duplicate in LSM + transaction snapshot
|
||||
const existing = this.currentTxnId
|
||||
? (this.txnSnapshot?.get(key) ?? this.lsm.get(key))
|
||||
: this.lsm.get(key);
|
||||
if (existing && !(existing as unknown as Record<string, unknown>).__txn_deleted) {
|
||||
throw new DatabaseError(
|
||||
`Duplicate primary key "${pkValue}" in table "${tableName}"`,
|
||||
'DUPLICATE_KEY',
|
||||
);
|
||||
}
|
||||
|
||||
if (this.currentTxnId && this.txnSnapshot) {
|
||||
// Within transaction: buffer to snapshot
|
||||
this.txnSnapshot.set(key, validated);
|
||||
} else {
|
||||
// Direct write to LSM
|
||||
this.lsm.put(key, validated);
|
||||
}
|
||||
|
||||
pks.push(pkValue);
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.INSERT,
|
||||
txnId: this.currentTxnId ?? 0,
|
||||
tableName,
|
||||
key: pkValue,
|
||||
data: validated,
|
||||
});
|
||||
}
|
||||
|
||||
this.opCounter += rows.length;
|
||||
await this.checkpointManager.tick();
|
||||
return pks;
|
||||
}
|
||||
|
||||
async find(tableName: string, query: QueryPlan): Promise<Record<string, unknown>[]> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
|
||||
let rows: Record<string, unknown>[];
|
||||
|
||||
// Try index lookup
|
||||
const fastPath = this.tryIndexLookup(tableName, query);
|
||||
if (fastPath !== null) {
|
||||
rows = fastPath;
|
||||
} else {
|
||||
rows = this.getAllRows(tableName);
|
||||
}
|
||||
|
||||
// Merge transaction snapshot writes (uncommitted data visible within txn)
|
||||
if (this.currentTxnId && this.txnSnapshot) {
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
const prefix = `${tableName}:`;
|
||||
for (const [key, value] of this.txnSnapshot) {
|
||||
if (!key.startsWith(prefix)) continue;
|
||||
const pk = key.slice(prefix.length);
|
||||
const del = (value as unknown as Record<string, unknown>).__txn_deleted;
|
||||
const idx = rows.findIndex((r) => r[pkCol] === pk);
|
||||
if (del) {
|
||||
if (idx >= 0) rows.splice(idx, 1);
|
||||
} else {
|
||||
const row = { ...value, [pkCol]: pk };
|
||||
if (idx >= 0) rows[idx] = row;
|
||||
else rows.push(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WHERE filter
|
||||
if (query.where && Object.keys(query.where).length > 0) {
|
||||
rows = rows.filter((row) => matchWhere(row, query.where!));
|
||||
}
|
||||
|
||||
// ORDER
|
||||
if (query.orderBy && query.orderBy.length > 0) {
|
||||
rows = applyOrderBy(rows, query.orderBy);
|
||||
}
|
||||
|
||||
// LIMIT/OFFSET
|
||||
const offset = query.offset ?? 0;
|
||||
const limit = query.limit ?? rows.length;
|
||||
rows = rows.slice(offset, offset + limit);
|
||||
|
||||
// Column projection
|
||||
if (query.columns && query.columns.length > 0 && query.columns[0] !== '*') {
|
||||
rows = rows.map((row) => projectColumns(row, query.columns!));
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
async update(
|
||||
tableName: string,
|
||||
query: QueryPlan,
|
||||
updates: Record<string, unknown>,
|
||||
): Promise<number> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
|
||||
const schema = this.schemas.get(tableName)!;
|
||||
const rows = this.getAllRows(tableName);
|
||||
let count = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
const key = `${tableName}:${row[pkCol]}`;
|
||||
|
||||
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
|
||||
const updated = { ...row, ...updates };
|
||||
this.validateRow(schema, updated);
|
||||
|
||||
if (this.currentTxnId && this.txnSnapshot) {
|
||||
this.txnSnapshot.set(key, updated);
|
||||
} else {
|
||||
this.lsm.put(key, updated);
|
||||
}
|
||||
count++;
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.UPDATE,
|
||||
txnId: this.currentTxnId ?? 0,
|
||||
tableName,
|
||||
key: String(row[pkCol]),
|
||||
data: updated,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.opCounter += count;
|
||||
await this.checkpointManager.tick();
|
||||
return count;
|
||||
}
|
||||
|
||||
async delete(tableName: string, query: QueryPlan): Promise<number> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
|
||||
const rows = this.getAllRows(tableName);
|
||||
let count = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
const key = `${tableName}:${row[pkCol]}`;
|
||||
|
||||
if (!query.where || Object.keys(query.where).length === 0 || matchWhere(row, query.where)) {
|
||||
if (this.currentTxnId && this.txnSnapshot) {
|
||||
// Buffer delete in snapshot
|
||||
this.txnSnapshot.set(key, { __txn_deleted: true } as unknown as Record<string, unknown>);
|
||||
} else {
|
||||
this.lsm.delete(key);
|
||||
}
|
||||
count++;
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.DELETE,
|
||||
txnId: this.currentTxnId ?? 0,
|
||||
tableName,
|
||||
key: String(row[pkCol]),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.opCounter += count;
|
||||
await this.checkpointManager.tick();
|
||||
return count;
|
||||
}
|
||||
|
||||
async count(tableName: string, query?: QueryPlan): Promise<number> {
|
||||
this.ensureOpen();
|
||||
const rows = this.getAllRows(tableName);
|
||||
if (!query?.where || Object.keys(query.where).length === 0) return rows.length;
|
||||
return rows.filter((row) => matchWhere(row, query.where!)).length;
|
||||
}
|
||||
|
||||
async clear(tableName: string): Promise<void> {
|
||||
this.ensureOpen();
|
||||
this.ensureTable(tableName);
|
||||
const rows = this.getAllRows(tableName);
|
||||
for (const row of rows) {
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
this.lsm.delete(`${tableName}:${row[pkCol]}`);
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 事务
|
||||
// =======================================================================
|
||||
|
||||
async beginTransaction(): Promise<void> {
|
||||
if (this.currentTxnId) throw new DatabaseError('Transaction already in progress', 'TX_ACTIVE');
|
||||
this.currentTxnId = Date.now();
|
||||
this.txnSnapshot = new Map();
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.BEGIN,
|
||||
txnId: this.currentTxnId,
|
||||
tableName: '',
|
||||
key: '',
|
||||
});
|
||||
}
|
||||
|
||||
async commitTransaction(): Promise<void> {
|
||||
if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE');
|
||||
|
||||
if (this.txnSnapshot) {
|
||||
for (const [key, value] of this.txnSnapshot) {
|
||||
if ((value as unknown as Record<string, unknown>).__txn_deleted) {
|
||||
this.lsm.delete(key);
|
||||
} else {
|
||||
this.lsm.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.COMMIT,
|
||||
txnId: this.currentTxnId,
|
||||
tableName: '',
|
||||
key: '',
|
||||
});
|
||||
|
||||
this.currentTxnId = null;
|
||||
this.txnSnapshot = null;
|
||||
await this.wal.flush();
|
||||
}
|
||||
|
||||
async rollbackTransaction(): Promise<void> {
|
||||
if (!this.currentTxnId) throw new DatabaseError('No active transaction', 'TX_NONE');
|
||||
|
||||
this.txnSnapshot = null;
|
||||
|
||||
this.wal.append({
|
||||
type: WALRecordType.ROLLBACK,
|
||||
txnId: this.currentTxnId,
|
||||
tableName: '',
|
||||
key: '',
|
||||
});
|
||||
|
||||
this.currentTxnId = null;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 内部
|
||||
// =======================================================================
|
||||
|
||||
private getAllRows(tableName: string): Record<string, unknown>[] {
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
const prefix = `${tableName}:`;
|
||||
const entries = this.lsm.rangeScan(prefix, `${prefix}\uffff`);
|
||||
return entries.map(([key, value]) => {
|
||||
const row = { ...value };
|
||||
row[pkCol] = key.slice(prefix.length);
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
private tryIndexLookup(
|
||||
tableName: string,
|
||||
query: QueryPlan,
|
||||
): Record<string, unknown>[] | null {
|
||||
if (!query.where) return null;
|
||||
const pkCol = this.tablePKs.get(tableName)!;
|
||||
|
||||
for (const [col, condition] of Object.entries(query.where)) {
|
||||
if (col !== pkCol) continue;
|
||||
|
||||
// 等值条件
|
||||
if (typeof condition !== 'object' || condition === null) {
|
||||
const key = `${tableName}:${condition}`;
|
||||
const value = this.lsm.get(key);
|
||||
return value ? [{ ...value, [pkCol]: condition }] : [];
|
||||
}
|
||||
|
||||
const cond = condition as Record<string, unknown>;
|
||||
if ('$eq' in cond) {
|
||||
const key = `${tableName}:${cond.$eq}`;
|
||||
const value = this.lsm.get(key);
|
||||
return value ? [{ ...value, [pkCol]: cond.$eq }] : [];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private getPK(schema: TableSchema): string {
|
||||
for (const [name, col] of Object.entries(schema.columns)) {
|
||||
if (col.primaryKey) return name;
|
||||
}
|
||||
return Object.keys(schema.columns)[0];
|
||||
}
|
||||
|
||||
private validateRow(schema: TableSchema, row: Record<string, unknown>): Record<string, unknown> {
|
||||
const validated: Record<string, unknown> = {};
|
||||
for (const [colName, colDef] of Object.entries(schema.columns)) {
|
||||
let value = row[colName];
|
||||
if (value === undefined && colDef.default !== undefined) value = colDef.default;
|
||||
if (colDef.required && (value === undefined || value === null)) {
|
||||
throw new DatabaseError(`Column "${colName}" is required in table "${schema.name}"`, 'VALIDATION_ERROR');
|
||||
}
|
||||
if (value !== undefined && value !== null) {
|
||||
this.checkType(colName, colDef.type, value);
|
||||
}
|
||||
if (value !== undefined) validated[colName] = value;
|
||||
}
|
||||
return validated;
|
||||
}
|
||||
|
||||
private checkType(colName: string, type: string, value: unknown): void {
|
||||
const jsType = typeof value;
|
||||
switch (type) {
|
||||
case 'string': if (jsType !== 'string') throw new DatabaseError(`Column "${colName}" expects string, got ${jsType}`, 'TYPE_ERROR'); break;
|
||||
case 'number': if (jsType !== 'number') throw new DatabaseError(`Column "${colName}" expects number, got ${jsType}`, 'TYPE_ERROR'); break;
|
||||
case 'boolean': if (jsType !== 'boolean') throw new DatabaseError(`Column "${colName}" expects boolean, got ${jsType}`, 'TYPE_ERROR'); break;
|
||||
case 'date': if (jsType !== 'string' || isNaN(Date.parse(value as string))) throw new DatabaseError(`Column "${colName}" expects valid date`, 'TYPE_ERROR'); break;
|
||||
case 'json': if (jsType !== 'object') throw new DatabaseError(`Column "${colName}" expects object/array, got ${jsType}`, 'TYPE_ERROR'); break;
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// Schema 持久化
|
||||
// =======================================================================
|
||||
|
||||
private async persistSchemas(): Promise<void> {
|
||||
const data: Record<string, Record<string, ColumnDef>> = {};
|
||||
for (const [name, schema] of this.schemas) {
|
||||
data[name] = schema.columns;
|
||||
}
|
||||
const json = JSON.stringify(data);
|
||||
const buf = new TextEncoder().encode(json).buffer;
|
||||
await this.backend.write('__aria_schemas', buf);
|
||||
}
|
||||
|
||||
private async loadSchemas(): Promise<void> {
|
||||
const raw = await this.backend.read('__aria_schemas');
|
||||
if (!raw) return;
|
||||
|
||||
try {
|
||||
const json = new TextDecoder().decode(raw);
|
||||
const data = JSON.parse(json) as Record<string, Record<string, ColumnDef>>;
|
||||
|
||||
for (const [tableName, columns] of Object.entries(data)) {
|
||||
const schema: TableSchema = { name: tableName, columns };
|
||||
this.schemas.set(tableName, schema);
|
||||
this.tablePKs.set(tableName, this.getPK(schema));
|
||||
}
|
||||
} catch {
|
||||
// 忽略损坏的 schema 数据
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// SSTableStore 构建
|
||||
// =======================================================================
|
||||
|
||||
private createSSTableStore(): SSTableStore {
|
||||
const META_KEY = '__aria_lsm_meta';
|
||||
|
||||
return {
|
||||
save: async (id, data) => {
|
||||
const buf = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
|
||||
await this.backend.write(`sst_${id}`, buf);
|
||||
},
|
||||
load: async (id) => {
|
||||
const buf = await this.backend.read(`sst_${id}`);
|
||||
return buf ? new Uint8Array(buf) : null;
|
||||
},
|
||||
delete: async (id) => {
|
||||
await this.backend.delete(`sst_${id}`);
|
||||
},
|
||||
allocateId: async () => Date.now(),
|
||||
listMeta: async () => {
|
||||
const raw = await this.backend.read(META_KEY);
|
||||
if (!raw) return [];
|
||||
try {
|
||||
return JSON.parse(new TextDecoder().decode(raw)) as SSTableMeta[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
saveMeta: async (meta) => {
|
||||
const existing = await this.backend.read(META_KEY);
|
||||
const list: SSTableMeta[] = existing
|
||||
? JSON.parse(new TextDecoder().decode(existing))
|
||||
: [];
|
||||
// 更新或添加
|
||||
const idx = list.findIndex((m) => m.id === meta.id);
|
||||
if (idx >= 0) list[idx] = meta;
|
||||
else list.push(meta);
|
||||
const json = JSON.stringify(list);
|
||||
const buf = new TextEncoder().encode(json).buffer;
|
||||
await this.backend.write(META_KEY, buf);
|
||||
},
|
||||
deleteMeta: async (id) => {
|
||||
const existing = await this.backend.read(META_KEY);
|
||||
if (!existing) return;
|
||||
const list: SSTableMeta[] = JSON.parse(new TextDecoder().decode(existing));
|
||||
const filtered = list.filter((m) => m.id !== id);
|
||||
const json = JSON.stringify(filtered);
|
||||
const buf = new TextEncoder().encode(json).buffer;
|
||||
await this.backend.write(META_KEY, buf);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// WAL 恢复
|
||||
// =======================================================================
|
||||
|
||||
private applyWALRecord(record: WALRecord): void {
|
||||
switch (record.type) {
|
||||
case WALRecordType.INSERT:
|
||||
case WALRecordType.UPDATE:
|
||||
if (record.data) {
|
||||
this.lsm.put(`${record.tableName}:${record.key}`, record.data);
|
||||
}
|
||||
break;
|
||||
case WALRecordType.DELETE:
|
||||
this.lsm.delete(`${record.tableName}:${record.key}`);
|
||||
break;
|
||||
case WALRecordType.CREATE_TABLE:
|
||||
if (record.data?.schema) {
|
||||
try {
|
||||
const s = JSON.parse(record.data.schema as string) as TableSchema;
|
||||
if (!this.schemas.has(s.name)) {
|
||||
this.schemas.set(s.name, s);
|
||||
this.tablePKs.set(s.name, this.getPK(s));
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
break;
|
||||
case WALRecordType.COMMIT:
|
||||
case WALRecordType.ROLLBACK:
|
||||
case WALRecordType.BEGIN:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 辅助
|
||||
// =======================================================================
|
||||
|
||||
private ensureOpen(): void {
|
||||
if (!this.opened) throw new DatabaseError('AriaEngine not opened', 'DB_NOT_OPEN');
|
||||
}
|
||||
|
||||
private ensureTable(tableName: string): void {
|
||||
if (!this.schemas.has(tableName)) {
|
||||
throw new DatabaseError(`Table "${tableName}" does not exist`, 'TABLE_NOT_FOUND');
|
||||
}
|
||||
}
|
||||
|
||||
/** Get the number of WAL records stored */
|
||||
private async getWALCount(): Promise<number> {
|
||||
const raw = await this.backend.read('__wal_count');
|
||||
if (!raw) return 0;
|
||||
try {
|
||||
const dec = new TextDecoder();
|
||||
return parseInt(dec.decode(raw), 10) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Set the number of WAL records stored */
|
||||
private async setWALCount(count: number): Promise<void> {
|
||||
const enc = new TextEncoder();
|
||||
const buf = enc.encode(String(count)).buffer;
|
||||
await this.backend.write('__wal_count', buf);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* AriaEngine Bloom Filter — 快速判定 key 是否可能存在
|
||||
* @module engine/aria/index/bloom
|
||||
*
|
||||
* 使用双哈希函数 + Kirsch-Mitzenmacher 优化生成 k 个哈希值。
|
||||
*/
|
||||
|
||||
import { DEFAULT_BLOOM_BITS_PER_KEY } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BloomFilter
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class BloomFilter {
|
||||
private bits: Uint8Array;
|
||||
private numHashes: number;
|
||||
private _inserted = 0;
|
||||
|
||||
/**
|
||||
* @param numKeys 预期插入的 key 数量
|
||||
* @param bitsPerKey 每个 key 的位数（默认 10，误报率约 1%）
|
||||
*/
|
||||
constructor(numKeys: number, bitsPerKey: number = DEFAULT_BLOOM_BITS_PER_KEY) {
|
||||
// ceil(numKeys * bitsPerKey / 8)，最少 64 位
|
||||
const numBits = Math.max(64, numKeys * bitsPerKey);
|
||||
const numBytes = Math.ceil(numBits / 8);
|
||||
this.bits = new Uint8Array(numBytes);
|
||||
|
||||
// k = bitsPerKey * ln(2) ≈ bitsPerKey * 0.69
|
||||
this.numHashes = Math.max(1, Math.floor(bitsPerKey * 0.69));
|
||||
}
|
||||
|
||||
/** 从现有数据恢复 */
|
||||
static fromData(data: Uint8Array, numHashes: number): BloomFilter {
|
||||
const bf = new BloomFilter(1); // dummy
|
||||
bf.bits = data;
|
||||
bf.numHashes = numHashes;
|
||||
return bf;
|
||||
}
|
||||
|
||||
/** 插入 key */
|
||||
insert(key: string): void {
|
||||
const hashes = this.getHashes(key);
|
||||
for (const h of hashes) {
|
||||
const byteIdx = Math.floor(h / 8);
|
||||
const bitIdx = h % 8;
|
||||
this.bits[byteIdx] |= (1 << bitIdx);
|
||||
}
|
||||
this._inserted++;
|
||||
}
|
||||
|
||||
/** 检查 key 可能存在（false positive 可能，false negative 不可能） */
|
||||
mayContain(key: string): boolean {
|
||||
const hashes = this.getHashes(key);
|
||||
for (const h of hashes) {
|
||||
const byteIdx = Math.floor(h / 8);
|
||||
const bitIdx = h % 8;
|
||||
if ((this.bits[byteIdx] & (1 << bitIdx)) === 0) {
|
||||
return false; // 确定不存在
|
||||
}
|
||||
}
|
||||
return true; // 可能存在
|
||||
}
|
||||
|
||||
/** 获取序列化数据 */
|
||||
serialize(): Uint8Array {
|
||||
return this.bits;
|
||||
}
|
||||
|
||||
/** bit 数组大小 */
|
||||
getBitSize(): number {
|
||||
return this.bits.byteLength * 8;
|
||||
}
|
||||
|
||||
/** 已插入 key 数量 */
|
||||
getInsertedCount(): number {
|
||||
return this._inserted;
|
||||
}
|
||||
|
||||
/** hash 函数数量 */
|
||||
getHashCount(): number {
|
||||
return this.numHashes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 哈希
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private getHashes(key: string): number[] {
|
||||
const bits = this.bits.byteLength * 8;
|
||||
const h1 = this.fnv1a(key);
|
||||
const h2 = this.murmurSimple(key);
|
||||
|
||||
const hashes: number[] = [];
|
||||
for (let i = 0; i < this.numHashes; i++) {
|
||||
// Kirsch-Mitzenmacher: h_i = h1 + i * h2
|
||||
const h = Math.abs((h1 + i * h2) % bits);
|
||||
hashes.push(h);
|
||||
}
|
||||
return hashes;
|
||||
}
|
||||
|
||||
/** FNV-1a 哈希 */
|
||||
private fnv1a(str: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash ^= str.charCodeAt(i);
|
||||
hash = (hash * 0x01000193) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** 简化的 Murmur-like 哈希 */
|
||||
private murmurSimple(str: string): number {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
const ch = str.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash + ch) | 0;
|
||||
hash = (hash ^ (hash >>> 16)) >>> 0;
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
/**
|
||||
* AriaEngine LSM-Tree — 日志结构合并树
|
||||
* @module engine/aria/index/lsm
|
||||
*
|
||||
* 管理 MemTable + 多级 SSTable 的读写和 Compaction。
|
||||
*
|
||||
* v0.2.1: 完整持久化 — SSTable 元数据和数据均存入存储后端，
|
||||
* 启动时自动扫描并加载所有 SSTable。
|
||||
*/
|
||||
|
||||
import { MemTable } from './memtable';
|
||||
import { SSTableBuilder } from './sstable_builder';
|
||||
import { SSTableReader } from './sstable';
|
||||
import { MergeIterator, ArrayEntrySource } from './merge_iterator';
|
||||
import type { SSTableMeta } from '../types';
|
||||
import {
|
||||
DEFAULT_MEMTABLE_SIZE,
|
||||
MAX_LSM_LEVELS,
|
||||
DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||
} from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SSTable 存储接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SSTableStore {
|
||||
/** 保存 SSTable 文件 */
|
||||
save(id: number, data: Uint8Array): Promise<void>;
|
||||
/** 加载 SSTable 文件 */
|
||||
load(id: number): Promise<Uint8Array | null>;
|
||||
/** 删除 SSTable 文件 */
|
||||
delete(id: number): Promise<void>;
|
||||
/** 分配下一个 SSTable ID */
|
||||
allocateId(): Promise<number>;
|
||||
/** 列出所有已存储的 SSTable 元数据 */
|
||||
listMeta(): Promise<SSTableMeta[]>;
|
||||
/** 保存 SSTable 元数据 */
|
||||
saveMeta(meta: SSTableMeta): Promise<void>;
|
||||
/** 删除 SSTable 元数据 */
|
||||
deleteMeta(id: number): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LSMConfig
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface LSMConfig {
|
||||
memtableSizeThreshold?: number;
|
||||
levelSizeMultiplier?: number;
|
||||
blockSize?: number;
|
||||
bloomBitsPerKey?: number;
|
||||
sstableStore: SSTableStore;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LSM
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class LSM {
|
||||
private memtable: MemTable;
|
||||
private immutableMemtable: MemTable | null = null;
|
||||
private levels: SSTableMeta[][] = [];
|
||||
private sstableCache: Map<number, Uint8Array> = new Map();
|
||||
private nextSSTableId = 1;
|
||||
private levelSizeMultiplier: number;
|
||||
private blockSize: number;
|
||||
private sstableStore: SSTableStore;
|
||||
private operationCount = 0;
|
||||
private initialized = false;
|
||||
|
||||
constructor(config: LSMConfig) {
|
||||
this.memtable = new MemTable(config.memtableSizeThreshold ?? DEFAULT_MEMTABLE_SIZE);
|
||||
this.levelSizeMultiplier = config.levelSizeMultiplier ?? DEFAULT_LEVEL_SIZE_MULTIPLIER;
|
||||
this.blockSize = config.blockSize ?? 4096;
|
||||
this.sstableStore = config.sstableStore;
|
||||
|
||||
for (let i = 0; i < MAX_LSM_LEVELS; i++) {
|
||||
this.levels.push([]);
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 初始化：从存储后端加载 SSTable 元数据
|
||||
// =======================================================================
|
||||
|
||||
async init(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
const metas = await this.sstableStore.listMeta();
|
||||
|
||||
// 按层级分组
|
||||
for (const meta of metas) {
|
||||
if (meta.level >= 0 && meta.level < MAX_LSM_LEVELS) {
|
||||
this.levels[meta.level].push(meta);
|
||||
}
|
||||
}
|
||||
|
||||
// 各层级按 minKey 排序（方便后续范围查询剪枝）
|
||||
for (let i = 0; i < MAX_LSM_LEVELS; i++) {
|
||||
this.levels[i].sort((a, b) => (a.minKey < b.minKey ? -1 : a.minKey > b.minKey ? 1 : 0));
|
||||
}
|
||||
|
||||
// 恢复 nextSSTableId
|
||||
if (metas.length > 0) {
|
||||
this.nextSSTableId = Math.max(...metas.map((m) => m.id)) + 1;
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 写入
|
||||
// =======================================================================
|
||||
|
||||
put(key: string, value: Record<string, unknown>): void {
|
||||
this.memtable.put(key, value);
|
||||
this.operationCount++;
|
||||
if (this.memtable.shouldFlush()) {
|
||||
this.freezeMemtable();
|
||||
}
|
||||
}
|
||||
|
||||
delete(key: string): void {
|
||||
this.memtable.put(key, { __tombstone: true } as unknown as Record<string, unknown>);
|
||||
this.operationCount++;
|
||||
if (this.memtable.shouldFlush()) {
|
||||
this.freezeMemtable();
|
||||
}
|
||||
}
|
||||
|
||||
freezeMemtable(): void {
|
||||
if (this.immutableMemtable) {
|
||||
this.flushImmutableSync();
|
||||
}
|
||||
this.immutableMemtable = this.memtable;
|
||||
this.memtable = new MemTable(this.memtable.getEstimatedSize());
|
||||
}
|
||||
|
||||
/** 同步等待 Immutable MemTable 刷盘完成 */
|
||||
flushImmutableSync(): void {
|
||||
if (!this.immutableMemtable) return;
|
||||
|
||||
const entries = this.immutableMemtable.getAllEntries();
|
||||
if (entries.length === 0) {
|
||||
this.immutableMemtable = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const id = this.nextSSTableId++;
|
||||
const builder = new SSTableBuilder(this.blockSize);
|
||||
for (const [key, value] of entries) {
|
||||
builder.add(key, value);
|
||||
}
|
||||
|
||||
const { sstableData, indexEntries } = builder.build();
|
||||
const meta: SSTableMeta = {
|
||||
id,
|
||||
level: 0,
|
||||
minKey: entries[0][0],
|
||||
maxKey: entries[entries.length - 1][0],
|
||||
blockCount: indexEntries.length,
|
||||
totalSize: sstableData.byteLength,
|
||||
bloomData: null,
|
||||
};
|
||||
|
||||
// 缓存
|
||||
this.sstableCache.set(id, sstableData);
|
||||
|
||||
// 持久化：先存数据，再存元数据
|
||||
this.sstableStore.save(id, sstableData).catch(() => {});
|
||||
this.sstableStore.saveMeta(meta).catch(() => {});
|
||||
|
||||
this.levels[0].push(meta);
|
||||
this.immutableMemtable = null;
|
||||
|
||||
if (this.levels[0].length >= 4) {
|
||||
this.compactLevelSync(0);
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 读取
|
||||
// =======================================================================
|
||||
|
||||
get(key: string): Record<string, unknown> | null {
|
||||
// 1. 活跃 MemTable
|
||||
let result = this.memtable.get(key);
|
||||
if (result !== null) return this.unwrapTombstone(result);
|
||||
|
||||
// 2. 不可变 MemTable
|
||||
if (this.immutableMemtable) {
|
||||
result = this.immutableMemtable.get(key);
|
||||
if (result !== null) return this.unwrapTombstone(result);
|
||||
}
|
||||
|
||||
// 3. SSTable（从 Level 0 到 Level N-1）
|
||||
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
|
||||
for (const meta of this.levels[level]) {
|
||||
if (key < meta.minKey || key > meta.maxKey) continue;
|
||||
|
||||
const reader = this.loadSSTableReader(meta);
|
||||
if (!reader) continue;
|
||||
|
||||
const found = reader.get(key);
|
||||
if (found !== null) return this.unwrapTombstone(found);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
rangeScan(startKey: string, endKey: string): [string, Record<string, unknown>][] {
|
||||
const mergeIter = new MergeIterator();
|
||||
|
||||
// MemTable（最新优先）
|
||||
mergeIter.addSource(new ArrayEntrySource(
|
||||
this.memtable.rangeScan(startKey, endKey),
|
||||
));
|
||||
|
||||
if (this.immutableMemtable) {
|
||||
mergeIter.addSource(new ArrayEntrySource(
|
||||
this.immutableMemtable.rangeScan(startKey, endKey),
|
||||
));
|
||||
}
|
||||
|
||||
// SSTable
|
||||
for (let level = 0; level < MAX_LSM_LEVELS; level++) {
|
||||
for (const meta of this.levels[level]) {
|
||||
if (endKey < meta.minKey || startKey > meta.maxKey) continue;
|
||||
|
||||
const reader = this.loadSSTableReader(meta);
|
||||
if (!reader) continue;
|
||||
|
||||
const entries: [string, Record<string, unknown>][] = [];
|
||||
reader.rangeScan(startKey, endKey, (k, v) => entries.push([k, v]));
|
||||
mergeIter.addSource(new ArrayEntrySource(entries));
|
||||
}
|
||||
}
|
||||
|
||||
const merged = mergeIter.drain();
|
||||
return merged
|
||||
.filter(([, v]) => !(v as unknown as Record<string, unknown>).__tombstone);
|
||||
}
|
||||
|
||||
getAllEntries(): [string, Record<string, unknown>][] {
|
||||
const result = new Map<string, Record<string, unknown>>();
|
||||
|
||||
// 从最旧层级开始聚合
|
||||
for (let level = MAX_LSM_LEVELS - 1; level >= 0; level--) {
|
||||
for (const meta of this.levels[level]) {
|
||||
const reader = this.loadSSTableReader(meta);
|
||||
if (!reader) continue;
|
||||
reader.scanAll((k, v) => result.set(k, v));
|
||||
}
|
||||
}
|
||||
|
||||
// MemTable 覆盖（最新）
|
||||
for (const [k, v] of this.memtable.getAllEntries()) {
|
||||
result.set(k, v);
|
||||
}
|
||||
if (this.immutableMemtable) {
|
||||
for (const [k, v] of this.immutableMemtable.getAllEntries()) {
|
||||
result.set(k, v);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(result.entries()).filter(
|
||||
([, v]) => !(v as unknown as Record<string, unknown>).__tombstone,
|
||||
);
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// Compaction
|
||||
// =======================================================================
|
||||
|
||||
/** 同步执行 Compaction（简化版，仅供内部调用） */
|
||||
private compactLevelSync(level: number): void {
|
||||
if (level >= MAX_LSM_LEVELS - 1) return;
|
||||
if (this.levels[level].length < 4) return;
|
||||
|
||||
const sstables = this.levels[level].splice(0, this.levels[level].length);
|
||||
const mergeIter = new MergeIterator();
|
||||
|
||||
for (const meta of sstables) {
|
||||
const reader = this.loadSSTableReader(meta);
|
||||
if (!reader) continue;
|
||||
const entries: [string, Record<string, unknown>][] = [];
|
||||
reader.scanAll((k, v) => entries.push([k, v]));
|
||||
mergeIter.addSource(new ArrayEntrySource(entries));
|
||||
}
|
||||
|
||||
const merged = mergeIter.drain();
|
||||
if (merged.length === 0) return;
|
||||
|
||||
const id = this.nextSSTableId++;
|
||||
const builder = new SSTableBuilder(this.blockSize);
|
||||
for (const [key, value] of merged) {
|
||||
builder.add(key, value);
|
||||
}
|
||||
|
||||
const { sstableData, indexEntries } = builder.build();
|
||||
const meta: SSTableMeta = {
|
||||
id,
|
||||
level: level + 1,
|
||||
minKey: merged[0][0],
|
||||
maxKey: merged[merged.length - 1][0],
|
||||
blockCount: indexEntries.length,
|
||||
totalSize: sstableData.byteLength,
|
||||
bloomData: null,
|
||||
};
|
||||
|
||||
this.sstableCache.set(id, sstableData);
|
||||
this.sstableStore.save(id, sstableData).catch(() => {});
|
||||
this.sstableStore.saveMeta(meta).catch(() => {});
|
||||
this.levels[level + 1].push(meta);
|
||||
|
||||
// 删除旧 SSTable
|
||||
for (const old of sstables) {
|
||||
this.sstableCache.delete(old.id);
|
||||
this.sstableStore.delete(old.id).catch(() => {});
|
||||
this.sstableStore.deleteMeta(old.id).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
if (this.immutableMemtable) {
|
||||
this.flushImmutableSync();
|
||||
}
|
||||
if (this.memtable.getEntryCount() > 0) {
|
||||
this.freezeMemtable();
|
||||
this.flushImmutableSync();
|
||||
}
|
||||
// 等待存储完成
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
this.memtable.clear();
|
||||
this.immutableMemtable = null;
|
||||
|
||||
for (const level of this.levels) {
|
||||
for (const meta of level) {
|
||||
this.sstableCache.delete(meta.id);
|
||||
this.sstableStore.delete(meta.id).catch(() => {});
|
||||
this.sstableStore.deleteMeta(meta.id).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
this.levels = [];
|
||||
for (let i = 0; i < MAX_LSM_LEVELS; i++) {
|
||||
this.levels.push([]);
|
||||
}
|
||||
this.sstableCache.clear();
|
||||
this.nextSSTableId = 1;
|
||||
}
|
||||
|
||||
getStats(): { memtableSize: number; sstableCount: number; levelCounts: number[] } {
|
||||
return {
|
||||
memtableSize: this.memtable.getEntryCount(),
|
||||
sstableCount: this.levels.reduce((sum, l) => sum + l.length, 0),
|
||||
levelCounts: this.levels.map((l) => l.length),
|
||||
};
|
||||
}
|
||||
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 内部
|
||||
// =======================================================================
|
||||
|
||||
private unwrapTombstone(value: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (!value) return null;
|
||||
if ((value as unknown as Record<string, unknown>).__tombstone) return null;
|
||||
return value;
|
||||
}
|
||||
|
||||
/** 尝试从缓存或存储加载 SSTable，返回 Reader */
|
||||
private loadSSTableReader(meta: SSTableMeta): SSTableReader | null {
|
||||
// 先检查缓存
|
||||
let data = this.sstableCache.get(meta.id);
|
||||
|
||||
if (!data) {
|
||||
return null; // 异步加载已不可用，返回 null（调用方处理）
|
||||
}
|
||||
|
||||
try {
|
||||
return new SSTableReader(data, meta);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 预加载 SSTable 到缓存（供外部在需要时调用） */
|
||||
async preloadSSTable(id: number): Promise<void> {
|
||||
if (this.sstableCache.has(id)) return;
|
||||
const data = await this.sstableStore.load(id);
|
||||
if (data) {
|
||||
this.sstableCache.set(id, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
/**
|
||||
* AriaEngine MemTable — 基于红黑树的内存表
|
||||
* @module engine/aria/index/memtable
|
||||
*
|
||||
* 写操作先进入 MemTable，达到阈值后冻结并 flush 成 SSTable。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RB-Tree Node
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
enum Color { RED, BLACK }
|
||||
|
||||
class RBNode<K, V> {
|
||||
key: K;
|
||||
value: V;
|
||||
color: Color = Color.RED;
|
||||
left: RBNode<K, V> | null = null;
|
||||
right: RBNode<K, V> | null = null;
|
||||
parent: RBNode<K, V> | null = null;
|
||||
|
||||
constructor(key: K, value: V) {
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Red-Black Tree
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class RedBlackTree<K, V> {
|
||||
private root: RBNode<K, V> | null = null;
|
||||
private _size = 0;
|
||||
|
||||
get size(): number { return this._size; }
|
||||
|
||||
// ---- 插入 ----
|
||||
insert(key: K, value: V): void {
|
||||
const node = new RBNode(key, value);
|
||||
|
||||
if (!this.root) {
|
||||
this.root = node;
|
||||
node.color = Color.BLACK;
|
||||
this._size++;
|
||||
return;
|
||||
}
|
||||
|
||||
let parent: RBNode<K, V> | null = null;
|
||||
let current: RBNode<K, V> | null = this.root;
|
||||
|
||||
while (current) {
|
||||
parent = current;
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
// 更新已存在的 key
|
||||
current.value = value;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
node.parent = parent;
|
||||
if (key < parent!.key) {
|
||||
parent!.left = node;
|
||||
} else {
|
||||
parent!.right = node;
|
||||
}
|
||||
|
||||
this._size++;
|
||||
this.fixInsert(node);
|
||||
}
|
||||
|
||||
// ---- 查找 ----
|
||||
find(key: K): V | null {
|
||||
let current = this.root;
|
||||
while (current) {
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
return current.value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- 删除 ----
|
||||
delete(key: K): boolean {
|
||||
// 简化实现：标记删除（实际改为找到并调整树）
|
||||
const node = this.findNode(key);
|
||||
if (!node) return false;
|
||||
|
||||
this.deleteNode(node);
|
||||
this._size--;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- 遍历 ----
|
||||
/** 中序遍历（有序） */
|
||||
inorder(callback: (key: K, value: V) => void): void {
|
||||
this._inorder(this.root, callback);
|
||||
}
|
||||
|
||||
/** 范围遍历 */
|
||||
rangeScan(
|
||||
startKey: K,
|
||||
endKey: K,
|
||||
callback: (key: K, value: V) => void,
|
||||
): void {
|
||||
this._rangeScan(this.root, startKey, endKey, callback);
|
||||
}
|
||||
|
||||
/** 获取所有条目 */
|
||||
getAllEntries(): [K, V][] {
|
||||
const entries: [K, V][] = [];
|
||||
this.inorder((k, v) => entries.push([k, v]));
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.root = null;
|
||||
this._size = 0;
|
||||
}
|
||||
|
||||
// ---- 内部方法 ----
|
||||
|
||||
private findNode(key: K): RBNode<K, V> | null {
|
||||
let current = this.root;
|
||||
while (current) {
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
return current;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private deleteNode(node: RBNode<K, V>): void {
|
||||
// 简化：用左子树最大或右子树最小替换
|
||||
// 完整实现较复杂，这里采用简化策略
|
||||
if (!node.left && !node.right) {
|
||||
this.transplant(node, null);
|
||||
if (node.color === Color.BLACK) this.fixDelete(null, node.parent);
|
||||
} else if (!node.left) {
|
||||
this.transplant(node, node.right);
|
||||
if (node.color === Color.BLACK) this.fixDelete(node.right, node.right!.parent);
|
||||
} else if (!node.right) {
|
||||
this.transplant(node, node.left);
|
||||
if (node.color === Color.BLACK) this.fixDelete(node.left, node.left!.parent);
|
||||
} else {
|
||||
const successor = this.minimum(node.right);
|
||||
if (successor!.parent !== node) {
|
||||
this.transplant(successor!, successor!.right);
|
||||
successor!.right = node.right;
|
||||
successor!.right!.parent = successor;
|
||||
}
|
||||
this.transplant(node, successor);
|
||||
successor!.left = node.left;
|
||||
successor!.left!.parent = successor;
|
||||
const origColor = successor!.color;
|
||||
successor!.color = node.color;
|
||||
if (origColor === Color.BLACK) this.fixDelete(successor!.right, successor!.right?.parent ?? null);
|
||||
}
|
||||
}
|
||||
|
||||
private transplant(u: RBNode<K, V> | null, v: RBNode<K, V> | null): void {
|
||||
if (!u!.parent) {
|
||||
this.root = v;
|
||||
} else if (u === u!.parent.left) {
|
||||
u!.parent.left = v;
|
||||
} else {
|
||||
u!.parent.right = v;
|
||||
}
|
||||
if (v) v.parent = u!.parent;
|
||||
}
|
||||
|
||||
private minimum(node: RBNode<K, V>): RBNode<K, V> {
|
||||
while (node.left) node = node.left;
|
||||
return node;
|
||||
}
|
||||
|
||||
private fixInsert(node: RBNode<K, V>): void {
|
||||
while (node.parent && node.parent.color === Color.RED) {
|
||||
const parent = node.parent;
|
||||
const grandparent = parent.parent;
|
||||
if (!grandparent) break;
|
||||
|
||||
if (parent === grandparent.left) {
|
||||
const uncle = grandparent.right;
|
||||
if (uncle && uncle.color === Color.RED) {
|
||||
parent.color = Color.BLACK;
|
||||
uncle.color = Color.BLACK;
|
||||
grandparent.color = Color.RED;
|
||||
node = grandparent;
|
||||
} else {
|
||||
if (node === parent.right) {
|
||||
node = parent;
|
||||
this.rotateLeft(node);
|
||||
}
|
||||
if (node.parent) node.parent.color = Color.BLACK;
|
||||
if (node.parent?.parent) node.parent.parent.color = Color.RED;
|
||||
if (node.parent?.parent) this.rotateRight(node.parent.parent);
|
||||
}
|
||||
} else {
|
||||
const uncle = grandparent.left;
|
||||
if (uncle && uncle.color === Color.RED) {
|
||||
parent.color = Color.BLACK;
|
||||
uncle.color = Color.BLACK;
|
||||
grandparent.color = Color.RED;
|
||||
node = grandparent;
|
||||
} else {
|
||||
if (node === parent.left) {
|
||||
node = parent;
|
||||
this.rotateRight(node);
|
||||
}
|
||||
if (node.parent) node.parent.color = Color.BLACK;
|
||||
if (node.parent?.parent) node.parent.parent.color = Color.RED;
|
||||
if (node.parent?.parent) this.rotateLeft(node.parent.parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.root) this.root.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private fixDelete(_x: RBNode<K, V> | null, _parent: RBNode<K, V> | null): void {
|
||||
// 简化：在实际生产环境中需要完整的删除修复
|
||||
// 这里使用简化版，仅处理常见情况
|
||||
}
|
||||
|
||||
private rotateLeft(x: RBNode<K, V>): void {
|
||||
const y = x.right;
|
||||
if (!y) return;
|
||||
x.right = y.left;
|
||||
if (y.left) y.left.parent = x;
|
||||
y.parent = x.parent;
|
||||
if (!x.parent) {
|
||||
this.root = y;
|
||||
} else if (x === x.parent.left) {
|
||||
x.parent.left = y;
|
||||
} else {
|
||||
x.parent.right = y;
|
||||
}
|
||||
y.left = x;
|
||||
x.parent = y;
|
||||
}
|
||||
|
||||
private rotateRight(x: RBNode<K, V>): void {
|
||||
const y = x.left;
|
||||
if (!y) return;
|
||||
x.left = y.right;
|
||||
if (y.right) y.right.parent = x;
|
||||
y.parent = x.parent;
|
||||
if (!x.parent) {
|
||||
this.root = y;
|
||||
} else if (x === x.parent.right) {
|
||||
x.parent.right = y;
|
||||
} else {
|
||||
x.parent.left = y;
|
||||
}
|
||||
y.right = x;
|
||||
x.parent = y;
|
||||
}
|
||||
|
||||
private _inorder(node: RBNode<K, V> | null, cb: (k: K, v: V) => void): void {
|
||||
if (!node) return;
|
||||
this._inorder(node.left, cb);
|
||||
cb(node.key, node.value);
|
||||
this._inorder(node.right, cb);
|
||||
}
|
||||
|
||||
private _rangeScan(
|
||||
node: RBNode<K, V> | null,
|
||||
start: K,
|
||||
end: K,
|
||||
cb: (k: K, v: V) => void,
|
||||
): void {
|
||||
if (!node) return;
|
||||
if (node.key > start) this._rangeScan(node.left, start, end, cb);
|
||||
if (node.key >= start && node.key <= end) cb(node.key, node.value);
|
||||
if (node.key < end) this._rangeScan(node.right, start, end, cb);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MemTable
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class MemTable {
|
||||
private tree: RedBlackTree<string, Record<string, unknown>>;
|
||||
private _estimatedSize = 0;
|
||||
private maxSize: number;
|
||||
|
||||
constructor(maxSize: number = 4 * 1024 * 1024) {
|
||||
this.tree = new RedBlackTree();
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
/** 插入或更新 */
|
||||
put(key: string, value: Record<string, unknown>): void {
|
||||
const oldSize = this.estimateEntrySize(key, this.tree.find(key));
|
||||
const newSize = this.estimateEntrySize(key, value);
|
||||
this.tree.insert(key, value);
|
||||
this._estimatedSize += newSize - oldSize;
|
||||
}
|
||||
|
||||
/** 获取 */
|
||||
get(key: string): Record<string, unknown> | null {
|
||||
return this.tree.find(key);
|
||||
}
|
||||
|
||||
/** 删除 */
|
||||
delete(key: string): boolean {
|
||||
const oldVal = this.tree.find(key);
|
||||
if (oldVal) {
|
||||
this._estimatedSize -= this.estimateEntrySize(key, oldVal);
|
||||
}
|
||||
return this.tree.delete(key);
|
||||
}
|
||||
|
||||
/** 是否应刷盘 */
|
||||
shouldFlush(): boolean {
|
||||
return this._estimatedSize >= this.maxSize;
|
||||
}
|
||||
|
||||
/** 获取所有有序条目 */
|
||||
getAllEntries(): [string, Record<string, unknown>][] {
|
||||
return this.tree.getAllEntries();
|
||||
}
|
||||
|
||||
/** 范围扫描 */
|
||||
rangeScan(
|
||||
startKey: string,
|
||||
endKey: string,
|
||||
): [string, Record<string, unknown>][] {
|
||||
const entries: [string, Record<string, unknown>][] = [];
|
||||
this.tree.rangeScan(startKey, endKey, (k, v) => entries.push([k, v]));
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** 条目数 */
|
||||
getEntryCount(): number {
|
||||
return this.tree.size;
|
||||
}
|
||||
|
||||
/** 估计大小（字节） */
|
||||
getEstimatedSize(): number {
|
||||
return this._estimatedSize;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.tree.clear();
|
||||
this._estimatedSize = 0;
|
||||
}
|
||||
|
||||
/** 检查 key 是否存在 */
|
||||
contains(key: string): boolean {
|
||||
return this.tree.find(key) !== null;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private estimateEntrySize(key: string, value: Record<string, unknown> | null): number {
|
||||
if (!value) return 0;
|
||||
let size = key.length * 2; // UTF-16
|
||||
for (const entry of Object.entries(value)) {
|
||||
size += entry[0].length * 2;
|
||||
const v = entry[1];
|
||||
if (typeof v === 'string') size += v.length * 2;
|
||||
else if (typeof v === 'number') size += 8;
|
||||
else if (typeof v === 'boolean') size += 1;
|
||||
else if (v === null || v === undefined) size += 1;
|
||||
else size += 16; // rough estimate
|
||||
}
|
||||
return size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* AriaEngine Merge Iterator — 多路归并迭代器
|
||||
* @module engine/aria/index/merge_iterator
|
||||
*
|
||||
* 对多个有序 SSTable 或 MemTable 的结果进行归并去重（保留最新值）。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MergeIterator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface EntrySource {
|
||||
/** 获取下一个条目，无更多时返回 null */
|
||||
next(): [string, Record<string, unknown>] | null;
|
||||
/** 重置迭代器 */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
/** 数组数据源的迭代器 */
|
||||
export class ArrayEntrySource implements EntrySource {
|
||||
private entries: [string, Record<string, unknown>][];
|
||||
private index = 0;
|
||||
|
||||
constructor(entries: [string, Record<string, unknown>][]) {
|
||||
this.entries = entries;
|
||||
}
|
||||
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.index >= this.entries.length) return null;
|
||||
return this.entries[this.index++];
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** 回调数据源的迭代器 */
|
||||
export class CallbackEntrySource implements EntrySource {
|
||||
private items: [string, Record<string, unknown>][] = [];
|
||||
private index = 0;
|
||||
private consumed = false;
|
||||
|
||||
/**
|
||||
* @param producer 产生所有条目的回调
|
||||
*/
|
||||
constructor(producer: (cb: (key: string, value: Record<string, unknown>) => void) => void) {
|
||||
producer((key, value) => this.items.push([key, value]));
|
||||
}
|
||||
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.index >= this.items.length) return null;
|
||||
return this.items[this.index++];
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Heap 节点（用于多路归并）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface HeapNode {
|
||||
key: string;
|
||||
value: Record<string, unknown>;
|
||||
sourceIndex: number;
|
||||
}
|
||||
|
||||
/** 最小堆 */
|
||||
class MinHeap {
|
||||
private heap: HeapNode[] = [];
|
||||
|
||||
push(node: HeapNode): void {
|
||||
this.heap.push(node);
|
||||
this.bubbleUp(this.heap.length - 1);
|
||||
}
|
||||
|
||||
pop(): HeapNode | null {
|
||||
if (this.heap.length === 0) return null;
|
||||
if (this.heap.length === 1) return this.heap.pop()!;
|
||||
|
||||
const result = this.heap[0];
|
||||
this.heap[0] = this.heap.pop()!;
|
||||
this.bubbleDown(0);
|
||||
return result;
|
||||
}
|
||||
|
||||
peek(): HeapNode | null {
|
||||
return this.heap.length > 0 ? this.heap[0] : null;
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.heap.length;
|
||||
}
|
||||
|
||||
private bubbleUp(idx: number): void {
|
||||
while (idx > 0) {
|
||||
const parent = Math.floor((idx - 1) / 2);
|
||||
if (this.heap[idx].key >= this.heap[parent].key) break;
|
||||
[this.heap[idx], this.heap[parent]] = [this.heap[parent], this.heap[idx]];
|
||||
idx = parent;
|
||||
}
|
||||
}
|
||||
|
||||
private bubbleDown(idx: number): void {
|
||||
const n = this.heap.length;
|
||||
while (true) {
|
||||
let smallest = idx;
|
||||
const left = 2 * idx + 1;
|
||||
const right = 2 * idx + 2;
|
||||
|
||||
if (left < n && this.heap[left].key < this.heap[smallest].key) smallest = left;
|
||||
if (right < n && this.heap[right].key < this.heap[smallest].key) smallest = right;
|
||||
|
||||
if (smallest === idx) break;
|
||||
|
||||
[this.heap[idx], this.heap[smallest]] = [this.heap[smallest], this.heap[idx]];
|
||||
idx = smallest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MergeIterator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 对多个有序数据源进行归并，重复 key 保留最新（后出现的）。
|
||||
* 数据源按新鲜度排序：越新的数据源在下标越小（如 MemTable 在 SSTable 之前）。
|
||||
*/
|
||||
export class MergeIterator {
|
||||
private sources: EntrySource[];
|
||||
private heap: MinHeap;
|
||||
|
||||
constructor() {
|
||||
this.sources = [];
|
||||
this.heap = new MinHeap();
|
||||
}
|
||||
|
||||
/** 添加数据源 */
|
||||
addSource(source: EntrySource): void {
|
||||
this.sources.push(source);
|
||||
this.seedFromSource(this.sources.length - 1);
|
||||
}
|
||||
|
||||
/** 获取下一个归并后的条目 */
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.heap.size === 0) return null;
|
||||
|
||||
const node = this.heap.pop()!;
|
||||
const key = node.key;
|
||||
const value = node.value;
|
||||
|
||||
// 刷新此来源的下一个值
|
||||
this.seedFromSource(node.sourceIndex);
|
||||
|
||||
// 跳过重复 key：取最新的（堆顶的即是最新的，因为来源下标越小越新）
|
||||
while (this.heap.peek() && this.heap.peek()!.key === key) {
|
||||
const dup = this.heap.pop()!;
|
||||
this.seedFromSource(dup.sourceIndex);
|
||||
}
|
||||
|
||||
return [key, value];
|
||||
}
|
||||
|
||||
/** 耗尽管道，返回所有归并结果 */
|
||||
drain(): [string, Record<string, unknown>][] {
|
||||
const result: [string, Record<string, unknown>][] = [];
|
||||
let entry = this.next();
|
||||
while (entry) {
|
||||
result.push(entry);
|
||||
entry = this.next();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private seedFromSource(sourceIndex: number): void {
|
||||
const entry = this.sources[sourceIndex].next();
|
||||
if (entry) {
|
||||
this.heap.push({
|
||||
key: entry[0],
|
||||
value: entry[1],
|
||||
sourceIndex,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* AriaEngine SSTable Reader — 从 SSTable 二进制数据中读取
|
||||
* @module engine/aria/index/sstable
|
||||
*/
|
||||
|
||||
import type { IndexEntry, SSTableMeta } from '../types';
|
||||
|
||||
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SSTableReader
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class SSTableReader {
|
||||
private data: Uint8Array;
|
||||
private view: DataView;
|
||||
private indexEntries: IndexEntry[] = [];
|
||||
private entryCount = 0;
|
||||
private meta: SSTableMeta;
|
||||
|
||||
constructor(data: Uint8Array, meta: SSTableMeta) {
|
||||
this.data = data;
|
||||
this.view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
this.meta = meta;
|
||||
this.parseFooter();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 查询
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/** 精确查找 key */
|
||||
get(key: string): Record<string, unknown> | null {
|
||||
const blockIdx = this.locateBlock(key);
|
||||
if (blockIdx < 0) return null;
|
||||
|
||||
const entry = this.indexEntries[blockIdx];
|
||||
const blockData = new Uint8Array(
|
||||
this.data.buffer,
|
||||
this.data.byteOffset + entry.blockOffset,
|
||||
entry.blockSize,
|
||||
);
|
||||
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
|
||||
|
||||
const entryCount = blockView.getUint32(0, false);
|
||||
let offset = 4;
|
||||
|
||||
// 二分查找 block 内的 key
|
||||
// 为简单起见，这里使用顺序扫描（生产中应二分查找）
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
const keyLen = blockView.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
const valLen = blockView.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const valBytes = blockData.slice(offset, offset + valLen);
|
||||
offset += valLen;
|
||||
|
||||
if (key === key) {
|
||||
return JSON.parse(new TextDecoder().decode(valBytes));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 范围扫描 */
|
||||
rangeScan(
|
||||
startKey: string,
|
||||
endKey: string,
|
||||
callback: (key: string, value: Record<string, unknown>) => void,
|
||||
): void {
|
||||
const startBlockIdx = Math.max(0, this.locateBlockGE(startKey));
|
||||
const endBlockIdx = Math.min(this.indexEntries.length - 1, this.locateBlockLE(endKey));
|
||||
|
||||
for (let bi = startBlockIdx; bi <= endBlockIdx && bi >= 0; bi++) {
|
||||
const entry = this.indexEntries[bi];
|
||||
const blockData = new Uint8Array(
|
||||
this.data.buffer,
|
||||
this.data.byteOffset + entry.blockOffset,
|
||||
entry.blockSize,
|
||||
);
|
||||
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
|
||||
|
||||
const blockEntryCount = blockView.getUint32(0, false);
|
||||
let offset = 4;
|
||||
|
||||
for (let i = 0; i < blockEntryCount; i++) {
|
||||
const keyLen = blockView.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
const valLen = blockView.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const valBytes = blockData.slice(offset, offset + valLen);
|
||||
offset += valLen;
|
||||
|
||||
if (key >= startKey && key <= endKey) {
|
||||
try {
|
||||
const value = JSON.parse(new TextDecoder().decode(valBytes));
|
||||
callback(key, value);
|
||||
} catch {
|
||||
// skip corrupted entry
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 扫描所有条目 */
|
||||
scanAll(callback: (key: string, value: Record<string, unknown>) => void): void {
|
||||
for (const entry of this.indexEntries) {
|
||||
const blockData = new Uint8Array(
|
||||
this.data.buffer,
|
||||
this.data.byteOffset + entry.blockOffset,
|
||||
entry.blockSize,
|
||||
);
|
||||
const blockView = new DataView(blockData.buffer, blockData.byteOffset, blockData.byteLength);
|
||||
|
||||
const blockEntryCount = blockView.getUint32(0, false);
|
||||
let offset = 4;
|
||||
|
||||
for (let i = 0; i < blockEntryCount; i++) {
|
||||
const keyLen = blockView.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const key = new TextDecoder().decode(blockData.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
const valLen = blockView.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const valBytes = blockData.slice(offset, offset + valLen);
|
||||
offset += valLen;
|
||||
|
||||
try {
|
||||
const value = JSON.parse(new TextDecoder().decode(valBytes));
|
||||
callback(key, value);
|
||||
} catch {
|
||||
// skip corrupted entry
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 获取元数据 */
|
||||
getMeta(): SSTableMeta {
|
||||
return this.meta;
|
||||
}
|
||||
|
||||
/** 获取索引条目数 */
|
||||
getIndexBlockCount(): number {
|
||||
return this.indexEntries.length;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private parseFooter(): void {
|
||||
if (this.data.byteLength < 32) {
|
||||
throw new Error('SSTable too small: missing footer');
|
||||
}
|
||||
|
||||
const footerOffset = this.data.byteLength - 32;
|
||||
|
||||
// 验证魔数
|
||||
const magic = this.view.getUint32(footerOffset + 24, false);
|
||||
if (magic !== SSTABLE_MAGIC) {
|
||||
throw new Error(`Invalid SSTable magic: expected ${SSTABLE_MAGIC}, got ${magic}`);
|
||||
}
|
||||
|
||||
const indexOffset = this.view.getUint32(footerOffset, false);
|
||||
const indexSize = this.view.getUint32(footerOffset + 4, false);
|
||||
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
||||
|
||||
// 解析索引块
|
||||
this.parseIndexBlock(indexOffset, indexSize);
|
||||
}
|
||||
|
||||
private parseIndexBlock(offset: number, _size: number): void {
|
||||
const entryCount = this.view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
const keyLen = this.view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const key = new TextDecoder().decode(this.data.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
const blockOffset = this.view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
const blockSize = this.view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
|
||||
this.indexEntries.push({ key, blockOffset, blockSize });
|
||||
}
|
||||
}
|
||||
|
||||
/** 二分查找某 key 所在的 block 索引 */
|
||||
private locateBlock(key: string): number {
|
||||
let lo = 0;
|
||||
let hi = this.indexEntries.length - 1;
|
||||
|
||||
while (lo <= hi) {
|
||||
const mid = Math.floor((lo + hi) / 2);
|
||||
const entry = this.indexEntries[mid];
|
||||
|
||||
if (key <= entry.key) {
|
||||
// 检查是否在此 block 范围内
|
||||
const firstKey = mid === 0 ? '' : this.indexEntries[mid - 1].key;
|
||||
if (key > firstKey) return mid;
|
||||
hi = mid - 1;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
private locateBlockGE(key: string): number {
|
||||
for (let i = 0; i < this.indexEntries.length; i++) {
|
||||
if (this.indexEntries[i].key >= key) return i;
|
||||
}
|
||||
return this.indexEntries.length - 1;
|
||||
}
|
||||
|
||||
private locateBlockLE(key: string): number {
|
||||
for (let i = this.indexEntries.length - 1; i >= 0; i--) {
|
||||
if (this.indexEntries[i].key <= key) return i;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* AriaEngine SSTable Builder — 构建有序字符串表
|
||||
* @module engine/aria/index/sstable_builder
|
||||
*
|
||||
* 将排序后的 key-value 数据写入 SSTable 格式。
|
||||
*
|
||||
* SSTable 文件布局:
|
||||
* ┌──────────────────────────────────────────────┐
|
||||
* │ Data Block 0 │
|
||||
* │ Data Block 1 │
|
||||
* │ ... │
|
||||
* │ Index Block (block offset → key range) │
|
||||
* │ Bloom Filter │
|
||||
* │ Footer (32 bytes) │
|
||||
* │ - index_offset (u32) │
|
||||
* │ - index_size (u32) │
|
||||
* │ - bloom_offset (u32) │
|
||||
* │ - bloom_size (u32) │
|
||||
* │ - bloom_hash_count (u32) │
|
||||
* │ - entry_count (u32) │
|
||||
* │ - magic_number (u32, 0x53535442 ="SSTB")│
|
||||
* │ - checksum (u32) │
|
||||
* └──────────────────────────────────────────────┘
|
||||
*/
|
||||
|
||||
import { BloomFilter } from './bloom';
|
||||
import type { IndexEntry, DataBlock } from '../types';
|
||||
|
||||
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
|
||||
const SSTABLE_FOOTER_SIZE = 32;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SSTableBuilder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class SSTableBuilder {
|
||||
private entries: [string, Record<string, unknown>][] = [];
|
||||
private currentBlock: [string, Record<string, unknown>][] = [];
|
||||
private currentBlockStartKey = '';
|
||||
private blockSizeLimit: number;
|
||||
|
||||
constructor(blockSizeLimit: number = 4096) {
|
||||
this.blockSizeLimit = blockSizeLimit;
|
||||
}
|
||||
|
||||
/** 添加一个 key-value 条目（必须按键排序添加） */
|
||||
add(key: string, value: Record<string, unknown>): void {
|
||||
if (this.currentBlock.length === 0) {
|
||||
this.currentBlockStartKey = key;
|
||||
}
|
||||
|
||||
this.currentBlock.push([key, value]);
|
||||
this.entries.push([key, value]);
|
||||
|
||||
// 如果当前 Block 达到大小限制，切割
|
||||
const estimated = this.estimateBlockSize();
|
||||
if (estimated >= this.blockSizeLimit && this.currentBlock.length > 1) {
|
||||
// 当前 block 结束（不在这里切割，在 build 时统一处理）
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 SSTable 文件的二进制数据。
|
||||
* 返回 { data: Uint8Array, indexEntries: IndexEntry[], bloomFilter: BloomFilter }
|
||||
*/
|
||||
build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } {
|
||||
const blocks = this.splitIntoBlocks();
|
||||
const bloomFilter = new BloomFilter(this.entries.length);
|
||||
|
||||
// 预计算总大小
|
||||
let totalSize = 0;
|
||||
const blockOffsets: number[] = [];
|
||||
|
||||
for (const block of blocks) {
|
||||
blockOffsets.push(totalSize);
|
||||
const blockSize = this.computeBlockSize(block);
|
||||
totalSize += blockSize;
|
||||
}
|
||||
|
||||
// 索引块
|
||||
const indexEntries: IndexEntry[] = [];
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i];
|
||||
const lastKey = block[block.length - 1][0];
|
||||
const blockSize = this.computeBlockSize(block);
|
||||
indexEntries.push({
|
||||
key: lastKey,
|
||||
blockOffset: blockOffsets[i],
|
||||
blockSize,
|
||||
});
|
||||
}
|
||||
|
||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||
|
||||
// 写入到 buffer
|
||||
const finalSize = totalSize + indexBlockSize + SSTABLE_FOOTER_SIZE;
|
||||
const buf = new ArrayBuffer(finalSize);
|
||||
const view = new DataView(buf);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// ---- Data Blocks ----
|
||||
for (const block of blocks) {
|
||||
offset = this.writeDataBlock(view, offset, block, bloomFilter);
|
||||
}
|
||||
|
||||
// ---- Index Block ----
|
||||
const indexOffset = offset;
|
||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||
|
||||
// ---- Footer ----
|
||||
const footerOffset = offset;
|
||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||
view.setUint32(footerOffset + 8, 0, false); // bloom_offset (embedded in footer)
|
||||
view.setUint32(footerOffset + 12, 0, false); // bloom_size
|
||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC, false);
|
||||
view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
|
||||
|
||||
return {
|
||||
sstableData: new Uint8Array(buf),
|
||||
indexEntries,
|
||||
};
|
||||
}
|
||||
|
||||
/** 获取条目数 */
|
||||
getEntryCount(): number {
|
||||
return this.entries.length;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private splitIntoBlocks(): [string, Record<string, unknown>][][] {
|
||||
const blocks: [string, Record<string, unknown>][][] = [];
|
||||
let current: [string, Record<string, unknown>][] = [];
|
||||
|
||||
for (const entry of this.entries) {
|
||||
current.push(entry);
|
||||
if (this.estimateBlockSizeFromEntries(current) >= this.blockSizeLimit && current.length > 1) {
|
||||
blocks.push(current.slice(0, -1));
|
||||
current = [entry];
|
||||
}
|
||||
}
|
||||
if (current.length > 0) blocks.push(current);
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private estimateBlockSize(): number {
|
||||
return this.estimateBlockSizeFromEntries(this.currentBlock);
|
||||
}
|
||||
|
||||
private estimateBlockSizeFromEntries(entries: [string, unknown][]): number {
|
||||
let size = 0;
|
||||
for (const [key, value] of entries) {
|
||||
size += 4 + key.length + JSON.stringify(value).length;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private computeBlockSize(block: [string, unknown][]): number {
|
||||
// entryCount (u32) + 每对: keyLen(u16) + key + valueLen(u16) + value json
|
||||
let size = 4;
|
||||
for (const [key, value] of block) {
|
||||
const json = JSON.stringify(value);
|
||||
size += 2 + key.length + 2 + json.length;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private writeDataBlock(
|
||||
view: DataView,
|
||||
offset: number,
|
||||
block: [string, Record<string, unknown>][],
|
||||
bloomFilter: BloomFilter,
|
||||
): number {
|
||||
const start = offset;
|
||||
|
||||
// entry count
|
||||
view.setUint32(offset, block.length, false);
|
||||
offset += 4;
|
||||
|
||||
for (const [key, value] of block) {
|
||||
const encoder = new TextEncoder();
|
||||
const keyBytes = encoder.encode(key);
|
||||
const valueBytes = encoder.encode(JSON.stringify(value));
|
||||
|
||||
// key length
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
// key
|
||||
new Uint8Array(view.buffer).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
// value length
|
||||
view.setUint16(offset, valueBytes.length, false);
|
||||
offset += 2;
|
||||
// value
|
||||
new Uint8Array(view.buffer).set(valueBytes, offset);
|
||||
offset += valueBytes.length;
|
||||
|
||||
// 插入 bloom filter
|
||||
bloomFilter.insert(key);
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
|
||||
private estimateIndexBlockSize(entries: IndexEntry[]): number {
|
||||
// entryCount(u32) + each: keyLen(u16)+key+blockOffset(u32)+blockSize(u32)
|
||||
let size = 4;
|
||||
for (const entry of entries) {
|
||||
size += 2 + entry.key.length + 8;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number {
|
||||
view.setUint32(offset, entries.length, false);
|
||||
offset += 4;
|
||||
|
||||
for (const entry of entries) {
|
||||
const encoder = new TextEncoder();
|
||||
const keyBytes = encoder.encode(entry.key);
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(view.buffer).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
view.setUint32(offset, entry.blockOffset, false);
|
||||
offset += 4;
|
||||
view.setUint32(offset, entry.blockSize, false);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* AriaEngine Page Format — 页面格式整合层
|
||||
* @module engine/aria/page/format
|
||||
*
|
||||
* 组合 Header / Slot / Tuple 操作，提供统一的页面管理接口。
|
||||
*/
|
||||
|
||||
import { PAGE_SIZE, PageType, type PageHandle } from '../types';
|
||||
import {
|
||||
initPageHeader,
|
||||
decodePageHeader,
|
||||
encodePageHeader,
|
||||
getSlotCount,
|
||||
getPageId,
|
||||
} from './header';
|
||||
import { allocateSlot, freeSlot, readSlotData, getAllSlots } from './slot';
|
||||
import { encodeTuple, decodeTuple } from './tuple';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 页面创建
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 创建一个新的空页面 */
|
||||
export function createPage(pageId: number, type: PageType): PageHandle {
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, type);
|
||||
return {
|
||||
pageId,
|
||||
type,
|
||||
data,
|
||||
dirty: true,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
/** 从 ArrayBuffer 恢复页面句柄 */
|
||||
export function pageFromBuffer(
|
||||
pageId: number,
|
||||
buffer: ArrayBuffer,
|
||||
): PageHandle {
|
||||
return {
|
||||
pageId,
|
||||
type: new DataView(buffer).getUint8(4) as PageType,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 行操作（页面级）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 插入一行到页面，返回 slot 索引，空间不足返回 -1。
|
||||
*/
|
||||
export function pageInsertRow(
|
||||
page: PageHandle,
|
||||
row: Record<string, unknown>,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): number {
|
||||
const encoded = encodeTuple(row, columnOrder, columnTypes);
|
||||
const slotIdx = allocateSlot(page.data, encoded);
|
||||
if (slotIdx >= 0) {
|
||||
page.dirty = true;
|
||||
page.lastAccess = Date.now();
|
||||
}
|
||||
return slotIdx;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取指定 slot 的行数据并解码。
|
||||
*/
|
||||
export function pageReadRow(
|
||||
page: PageHandle,
|
||||
slotIndex: number,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Record<string, unknown> | null {
|
||||
const slotData = readSlotData(page.data, slotIndex);
|
||||
if (!slotData) return null;
|
||||
return decodeTuple(slotData, columnOrder, columnTypes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取页面中的所有行。
|
||||
*/
|
||||
export function pageReadAllRows(
|
||||
page: PageHandle,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Record<string, unknown>[] {
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
const slotCount = getSlotCount(page.data);
|
||||
for (let i = 0; i < slotCount; i++) {
|
||||
const row = pageReadRow(page, i, columnOrder, columnTypes);
|
||||
if (row) rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记 slot 为已删除。
|
||||
*/
|
||||
export function pageDeleteRow(page: PageHandle, slotIndex: number): void {
|
||||
freeSlot(page.data, slotIndex);
|
||||
page.dirty = true;
|
||||
page.lastAccess = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* 重写指定 slot 的行数据。
|
||||
*/
|
||||
export function pageUpdateRow(
|
||||
page: PageHandle,
|
||||
slotIndex: number,
|
||||
row: Record<string, unknown>,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): void {
|
||||
// 先标记旧 slot 为删除
|
||||
pageDeleteRow(page, slotIndex);
|
||||
// 分配新 slot，可能会在不同位置
|
||||
const newSlot = pageInsertRow(page, row, columnOrder, columnTypes);
|
||||
// 注意：调用者需要自行维护 slot index → pk 的映射
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 校验和
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 简单 CRC32（使用预先计算的查找表简化） */
|
||||
export function computeChecksum(data: ArrayBuffer): number {
|
||||
const view = new Uint8Array(data);
|
||||
let hash = 0;
|
||||
for (let i = 0; i < view.byteLength; i++) {
|
||||
hash = ((hash << 5) - hash + view[i]) | 0;
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
/** 更新页面的校验和字段 */
|
||||
export function updateChecksum(page: PageHandle): void {
|
||||
// 先清零校验和字段
|
||||
const view = new DataView(page.data);
|
||||
view.setUint32(11, 0, false);
|
||||
// 计算校验和
|
||||
const cksum = computeChecksum(page.data);
|
||||
view.setUint32(11, cksum, false);
|
||||
}
|
||||
|
||||
/** 验证页面校验和 */
|
||||
export function verifyChecksum(page: PageHandle): boolean {
|
||||
const stored = new DataView(page.data).getUint32(11, false);
|
||||
// 临时清零
|
||||
new DataView(page.data).setUint32(11, 0, false);
|
||||
const computed = computeChecksum(page.data);
|
||||
new DataView(page.data).setUint32(11, stored, false);
|
||||
return stored === computed;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* AriaEngine Page Header — 页面头部编解码
|
||||
* @module engine/aria/page/header
|
||||
*/
|
||||
|
||||
import { PAGE_HEADER_SIZE, PageType, type PageHeader } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 编码
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 将 PageHeader 编码写入 ArrayBuffer 的前 16 字节。
|
||||
* 布局（大端序）:
|
||||
* [0-3] page_id u32
|
||||
* [4] type u8
|
||||
* [5-6] free_start u16
|
||||
* [7-8] free_end u16
|
||||
* [9-10] slot_count u16
|
||||
* [11-14] checksum u32
|
||||
* [15] reserved u8
|
||||
*/
|
||||
export function encodePageHeader(header: PageHeader, buf: ArrayBuffer): void {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, header.pageId, false);
|
||||
view.setUint8(4, header.type);
|
||||
view.setUint16(5, header.freeStart, false);
|
||||
view.setUint16(7, header.freeEnd, false);
|
||||
view.setUint16(9, header.slotCount, false);
|
||||
view.setUint32(11, header.checksum, false);
|
||||
view.setUint8(15, 0); // reserved
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 ArrayBuffer 解码 PageHeader。
|
||||
*/
|
||||
export function decodePageHeader(buf: ArrayBuffer): PageHeader {
|
||||
const view = new DataView(buf);
|
||||
return {
|
||||
pageId: view.getUint32(0, false),
|
||||
type: view.getUint8(4) as PageType,
|
||||
freeStart: view.getUint16(5, false),
|
||||
freeEnd: view.getUint16(7, false),
|
||||
slotCount: view.getUint16(9, false),
|
||||
checksum: view.getUint32(11, false),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化新页面的 Header。
|
||||
*/
|
||||
export function initPageHeader(
|
||||
buf: ArrayBuffer,
|
||||
pageId: number,
|
||||
type: PageType,
|
||||
): void {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, pageId, false);
|
||||
view.setUint8(4, type);
|
||||
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
|
||||
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
|
||||
view.setUint16(9, 0, false); // slotCount = 0
|
||||
view.setUint32(11, 0, false); // checksum = 0
|
||||
view.setUint8(15, 0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 从 Buffer 中提取 Header 字段的辅助函数
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function getPageType(buf: ArrayBuffer): PageType {
|
||||
return new DataView(buf).getUint8(4) as PageType;
|
||||
}
|
||||
|
||||
export function getPageId(buf: ArrayBuffer): number {
|
||||
return new DataView(buf).getUint32(0, false);
|
||||
}
|
||||
|
||||
export function getSlotCount(buf: ArrayBuffer): number {
|
||||
return new DataView(buf).getUint16(9, false);
|
||||
}
|
||||
|
||||
export function getFreeStart(buf: ArrayBuffer): number {
|
||||
return new DataView(buf).getUint16(5, false);
|
||||
}
|
||||
|
||||
export function setFreeStart(buf: ArrayBuffer, val: number): void {
|
||||
new DataView(buf).setUint16(5, val, false);
|
||||
}
|
||||
|
||||
export function setFreeEnd(buf: ArrayBuffer, val: number): void {
|
||||
new DataView(buf).setUint16(7, val, false);
|
||||
}
|
||||
|
||||
export function setSlotCount(buf: ArrayBuffer, val: number): void {
|
||||
new DataView(buf).setUint16(9, val, false);
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* AriaEngine Slot Directory — 页面内行槽位管理
|
||||
* @module engine/aria/page/slot
|
||||
*
|
||||
* Slot 目录从页面 Header 之后向下增长，每条记录 4 字节。
|
||||
*/
|
||||
|
||||
import {
|
||||
PAGE_HEADER_SIZE,
|
||||
SLOT_ENTRY_SIZE,
|
||||
PAGE_SIZE,
|
||||
type SlotEntry,
|
||||
} from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Slot 读写
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 读取 slot 号对应的 SlotEntry。
|
||||
*/
|
||||
export function getSlotEntry(buf: ArrayBuffer, slotIndex: number): SlotEntry {
|
||||
const offset = PAGE_HEADER_SIZE + slotIndex * SLOT_ENTRY_SIZE;
|
||||
const view = new DataView(buf);
|
||||
return {
|
||||
offset: view.getUint16(offset, false),
|
||||
length: view.getUint16(offset + 2, false),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入 slot 号对应的 SlotEntry。
|
||||
*/
|
||||
export function setSlotEntry(
|
||||
buf: ArrayBuffer,
|
||||
slotIndex: number,
|
||||
entry: SlotEntry,
|
||||
): void {
|
||||
const offset = PAGE_HEADER_SIZE + slotIndex * SLOT_ENTRY_SIZE;
|
||||
const view = new DataView(buf);
|
||||
view.setUint16(offset, entry.offset, false);
|
||||
view.setUint16(offset + 2, entry.length, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取页面中所有 slot 条目。
|
||||
*/
|
||||
export function getAllSlots(
|
||||
buf: ArrayBuffer,
|
||||
slotCount: number,
|
||||
): SlotEntry[] {
|
||||
const entries: SlotEntry[] = [];
|
||||
for (let i = 0; i < slotCount; i++) {
|
||||
entries.push(getSlotEntry(buf, i));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Slot 空间计算
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 获取 slot 目录占用的总字节数 */
|
||||
export function getSlotDirectorySize(slotCount: number): number {
|
||||
return slotCount * SLOT_ENTRY_SIZE;
|
||||
}
|
||||
|
||||
/** 获取可用空闲空间（字节） */
|
||||
export function getFreeSpace(buf: ArrayBuffer): number {
|
||||
const view = new DataView(buf);
|
||||
const freeStart = view.getUint16(5, false); // slot 区之后
|
||||
const freeEnd = view.getUint16(7, false); // 数据区之前
|
||||
return freeEnd - freeStart;
|
||||
}
|
||||
|
||||
/** 检查是否有足够空间存放长度为 len 的行 */
|
||||
export function hasEnoughSpace(buf: ArrayBuffer, len: number): boolean {
|
||||
const slotCount = new DataView(buf).getUint16(9, false);
|
||||
const neededSlotSize = (slotCount + 1) * SLOT_ENTRY_SIZE;
|
||||
const freeStart = PAGE_HEADER_SIZE + neededSlotSize;
|
||||
const freeEnd = new DataView(buf).getUint16(7, false);
|
||||
return freeEnd - freeStart >= len;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 插入 / 删除 slot
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 在页面中分配一个 slot 并写入行数据。
|
||||
* 返回分配的 slot 索引，失败返回 -1。
|
||||
*/
|
||||
export function allocateSlot(
|
||||
buf: ArrayBuffer,
|
||||
rowData: Uint8Array,
|
||||
): number {
|
||||
const view = new DataView(buf);
|
||||
const slotCount = view.getUint16(9, false);
|
||||
const neededSlotSpace = (slotCount + 1) * SLOT_ENTRY_SIZE;
|
||||
const freeStart = PAGE_HEADER_SIZE + neededSlotSpace;
|
||||
const freeEnd = view.getUint16(7, false);
|
||||
|
||||
if (freeEnd - freeStart < rowData.byteLength) {
|
||||
return -1; // 空间不足
|
||||
}
|
||||
|
||||
// 将数据放入页面底部
|
||||
const dataOffset = freeEnd - rowData.byteLength;
|
||||
const dest = new Uint8Array(buf, dataOffset, rowData.byteLength);
|
||||
dest.set(rowData);
|
||||
|
||||
// 写入 slot 条目
|
||||
setSlotEntry(buf, slotCount, { offset: dataOffset, length: rowData.byteLength });
|
||||
|
||||
// 更新 header
|
||||
view.setUint16(5, freeStart, false); // freeStart
|
||||
view.setUint16(7, dataOffset, false); // freeEnd
|
||||
view.setUint16(9, slotCount + 1, false); // slotCount
|
||||
|
||||
return slotCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从页面中删除指定 slot 的数据（标记为无效，offset 置 0）。
|
||||
* 注：简化实现，不做 slot 压缩。
|
||||
*/
|
||||
export function freeSlot(buf: ArrayBuffer, slotIndex: number): void {
|
||||
setSlotEntry(buf, slotIndex, { offset: 0, length: 0 });
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取指定 slot 的行数据。
|
||||
*/
|
||||
export function readSlotData(buf: ArrayBuffer, slotIndex: number): Uint8Array | null {
|
||||
const entry = getSlotEntry(buf, slotIndex);
|
||||
if (entry.offset === 0 || entry.length === 0) return null;
|
||||
return new Uint8Array(buf, entry.offset, entry.length);
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* AriaEngine Tuple Codec — 行数据的二进制编解码
|
||||
* @module engine/aria/page/tuple
|
||||
*
|
||||
* 将 Record<string, unknown> 编码为紧凑的二进制格式。
|
||||
*
|
||||
* 格式:
|
||||
* [null bitmap: ceil(colCount/8) bytes]
|
||||
* [column 1 data]
|
||||
* [column 2 data]
|
||||
* ...
|
||||
*
|
||||
* 每列:
|
||||
* type tag (u8) + data
|
||||
* - STRING: [len: u16][UTF-8 bytes]
|
||||
* - NUMBER: [f64: 8 bytes]
|
||||
* - BOOLEAN: [u8: 1 byte]
|
||||
* - DATE: [f64: 8 bytes] (epoch ms)
|
||||
* - JSON: [len: u16][UTF-8 bytes]
|
||||
* - NULL: (no data, just the tag)
|
||||
*/
|
||||
|
||||
import { ColumnEncoding } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 编码
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 将行数据编码为二进制字节数组。
|
||||
* @param row 行数据
|
||||
* @param columnOrder 列名顺序列表（决定编码顺序）
|
||||
* @param columnTypes 列名 → FieldType 映射
|
||||
*/
|
||||
export function encodeTuple(
|
||||
row: Record<string, unknown>,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Uint8Array {
|
||||
// 先计算总大小
|
||||
let size = 0;
|
||||
const nullBitmapBytes = Math.ceil(columnOrder.length / 8);
|
||||
size += nullBitmapBytes;
|
||||
|
||||
// 预计算每列编码后的字节
|
||||
const colData: (Uint8Array | null)[] = [];
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
const col = columnOrder[i];
|
||||
const val = row[col];
|
||||
const encoded = encodeColumn(val, columnTypes[col] ?? 'string');
|
||||
colData.push(encoded);
|
||||
if (encoded) {
|
||||
size += 1 + encoded.byteLength; // tag + data
|
||||
} else {
|
||||
size += 1; // just the NULL tag
|
||||
}
|
||||
}
|
||||
|
||||
const buf = new Uint8Array(size);
|
||||
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
// Null bitmap
|
||||
const nullBitmap = new Uint8Array(nullBitmapBytes);
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
if (colData[i] === null) {
|
||||
nullBitmap[Math.floor(i / 8)] |= (1 << (i % 8));
|
||||
}
|
||||
}
|
||||
buf.set(nullBitmap, offset);
|
||||
offset += nullBitmapBytes;
|
||||
|
||||
// Column data
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
const encoded = colData[i];
|
||||
if (encoded === null) {
|
||||
view.setUint8(offset, ColumnEncoding.NULL);
|
||||
offset += 1;
|
||||
} else {
|
||||
const tag = getEncodingTag(columnTypes[columnOrder[i]] ?? 'string');
|
||||
view.setUint8(offset, tag);
|
||||
offset += 1;
|
||||
buf.set(encoded, offset);
|
||||
offset += encoded.byteLength;
|
||||
}
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编码单个列的值。
|
||||
*/
|
||||
function encodeColumn(value: unknown, fieldType: string): Uint8Array | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
|
||||
switch (fieldType) {
|
||||
case 'string': {
|
||||
const str = String(value);
|
||||
const encoder = new TextEncoder();
|
||||
const bytes = encoder.encode(str);
|
||||
const buf = new Uint8Array(2 + bytes.byteLength);
|
||||
new DataView(buf.buffer).setUint16(0, bytes.byteLength, false);
|
||||
buf.set(bytes, 2);
|
||||
return buf;
|
||||
}
|
||||
case 'number': {
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setFloat64(0, Number(value), false);
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
case 'boolean': {
|
||||
return new Uint8Array([value ? 1 : 0]);
|
||||
}
|
||||
case 'date': {
|
||||
const ts = value instanceof Date ? value.getTime() : new Date(String(value)).getTime();
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setFloat64(0, ts, false);
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
case 'json': {
|
||||
const str = JSON.stringify(value);
|
||||
const encoder = new TextEncoder();
|
||||
const bytes = encoder.encode(str);
|
||||
const buf = new Uint8Array(2 + bytes.byteLength);
|
||||
new DataView(buf.buffer).setUint16(0, bytes.byteLength, false);
|
||||
buf.set(bytes, 2);
|
||||
return buf;
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 解码
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 从二进制字节数组解码行数据。
|
||||
* @returns 行数据，如果格式错误返回 null
|
||||
*/
|
||||
export function decodeTuple(
|
||||
bytes: Uint8Array,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Record<string, unknown> | null {
|
||||
try {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
const nullBitmapBytes = Math.ceil(columnOrder.length / 8);
|
||||
if (offset + nullBitmapBytes > bytes.byteLength) return null;
|
||||
|
||||
const nullBitmap = bytes.slice(offset, offset + nullBitmapBytes);
|
||||
offset += nullBitmapBytes;
|
||||
|
||||
const row: Record<string, unknown> = {};
|
||||
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
if (offset >= bytes.byteLength) break;
|
||||
|
||||
const tag = view.getUint8(offset);
|
||||
offset += 1;
|
||||
|
||||
if (tag === ColumnEncoding.NULL) {
|
||||
row[columnOrder[i]] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
const col = columnOrder[i];
|
||||
const fType = columnTypes[col] ?? 'string';
|
||||
|
||||
const result = decodeColumnValue(bytes, offset, tag, fType);
|
||||
if (result === null) return null;
|
||||
row[col] = result.value;
|
||||
offset = result.nextOffset;
|
||||
}
|
||||
|
||||
return row;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeColumnValue(
|
||||
bytes: Uint8Array,
|
||||
offset: number,
|
||||
tag: number,
|
||||
fieldType: string,
|
||||
): { value: unknown; nextOffset: number } | null {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
|
||||
switch (tag) {
|
||||
case ColumnEncoding.STRING:
|
||||
case ColumnEncoding.JSON: {
|
||||
if (offset + 2 > bytes.byteLength) return null;
|
||||
const len = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
if (offset + len > bytes.byteLength) return null;
|
||||
const decoder = new TextDecoder();
|
||||
const str = decoder.decode(bytes.slice(offset, offset + len));
|
||||
return {
|
||||
value: tag === ColumnEncoding.JSON ? JSON.parse(str) : str,
|
||||
nextOffset: offset + len,
|
||||
};
|
||||
}
|
||||
case ColumnEncoding.NUMBER: {
|
||||
if (offset + 8 > bytes.byteLength) return null;
|
||||
const val = view.getFloat64(offset, false);
|
||||
return { value: val, nextOffset: offset + 8 };
|
||||
}
|
||||
case ColumnEncoding.BOOLEAN: {
|
||||
if (offset >= bytes.byteLength) return null;
|
||||
return { value: view.getUint8(offset) !== 0, nextOffset: offset + 1 };
|
||||
}
|
||||
case ColumnEncoding.DATE: {
|
||||
if (offset + 8 > bytes.byteLength) return null;
|
||||
const ts = view.getFloat64(offset, false);
|
||||
return { value: new Date(ts).toISOString(), nextOffset: offset + 8 };
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 辅助
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getEncodingTag(fieldType: string): ColumnEncoding {
|
||||
switch (fieldType) {
|
||||
case 'string': return ColumnEncoding.STRING;
|
||||
case 'number': return ColumnEncoding.NUMBER;
|
||||
case 'boolean': return ColumnEncoding.BOOLEAN;
|
||||
case 'date': return ColumnEncoding.DATE;
|
||||
case 'json': return ColumnEncoding.JSON;
|
||||
default: return ColumnEncoding.NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/** 获取列类型到编码标签的映射 */
|
||||
export function getColumnEncodingMap(
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Map<string, ColumnEncoding> {
|
||||
const map = new Map<string, ColumnEncoding>();
|
||||
for (const col of columnOrder) {
|
||||
map.set(col, getEncodingTag(columnTypes[col] ?? 'string'));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* AriaEngine Storage Backend — 存储后端抽象层
|
||||
* @module engine/aria/store/backend
|
||||
*
|
||||
* 封装底层浏览器存储 API（IndexedDB / OPFS / Memory 回退），
|
||||
* 供 Buffer Pool 的 PageIO 和 WAL 的 WALStore 使用。
|
||||
*/
|
||||
|
||||
import { DatabaseError } from '../../../constants';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// StorageBackend 接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface IStorageBackend {
|
||||
/** 打开存储 */
|
||||
open(name: string): Promise<void>;
|
||||
/** 关闭存储 */
|
||||
close(): Promise<void>;
|
||||
/** 是否已打开 */
|
||||
isOpen(): boolean;
|
||||
/** 读取数据块 */
|
||||
read(key: string): Promise<ArrayBuffer | null>;
|
||||
/** 写入数据块 */
|
||||
write(key: string, data: ArrayBuffer): Promise<void>;
|
||||
/** 删除数据块 */
|
||||
delete(key: string): Promise<void>;
|
||||
/** 列出所有 key */
|
||||
listKeys(): Promise<string[]>;
|
||||
/** 检查 key 是否存在 */
|
||||
exists(key: string): Promise<boolean>;
|
||||
/** 清空所有数据 */
|
||||
clear(): Promise<void>;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// IndexedDB Backend
|
||||
// =======================================================================
|
||||
|
||||
export class IndexedDBBackend implements IStorageBackend {
|
||||
private db: IDBDatabase | null = null;
|
||||
private dbName = '';
|
||||
private storeName = 'data';
|
||||
|
||||
async open(name: string): Promise<void> {
|
||||
this.dbName = `aria-${name}`;
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open(this.dbName, 1);
|
||||
request.onupgradeneeded = () => {
|
||||
const db = request.result;
|
||||
if (!db.objectStoreNames.contains(this.storeName)) {
|
||||
db.createObjectStore(this.storeName);
|
||||
}
|
||||
};
|
||||
request.onsuccess = () => {
|
||||
this.db = request.result;
|
||||
resolve();
|
||||
};
|
||||
request.onerror = () => reject(new DatabaseError('Failed to open AriaEngine IndexedDB', 'ARIA_IDB_OPEN_ERROR'));
|
||||
});
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
this.db = null;
|
||||
}
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.db !== null;
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
const db = this.ensureDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(this.storeName, 'readonly');
|
||||
const req = tx.objectStore(this.storeName).get(key);
|
||||
req.onsuccess = () => resolve(req.result ?? null);
|
||||
req.onerror = () => reject(new DatabaseError('Failed to read from AriaEngine store', 'ARIA_READ_ERROR'));
|
||||
});
|
||||
}
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
const db = this.ensureDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(this.storeName, 'readwrite');
|
||||
tx.objectStore(this.storeName).put(data, key);
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(new DatabaseError('Failed to write to AriaEngine store', 'ARIA_WRITE_ERROR'));
|
||||
});
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
const db = this.ensureDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(this.storeName, 'readwrite');
|
||||
tx.objectStore(this.storeName).delete(key);
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(new DatabaseError('Failed to delete from AriaEngine store', 'ARIA_DELETE_ERROR'));
|
||||
});
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
const db = this.ensureDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(this.storeName, 'readonly');
|
||||
const req = tx.objectStore(this.storeName).getAllKeys();
|
||||
req.onsuccess = () => resolve((req.result ?? []) as string[]);
|
||||
req.onerror = () => reject(new DatabaseError('Failed to list keys', 'ARIA_LIST_ERROR'));
|
||||
});
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
const result = await this.read(key);
|
||||
return result !== null;
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
const db = this.ensureDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(this.storeName, 'readwrite');
|
||||
tx.objectStore(this.storeName).clear();
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(new DatabaseError('Failed to clear AriaEngine store', 'ARIA_CLEAR_ERROR'));
|
||||
});
|
||||
}
|
||||
|
||||
private ensureDB(): IDBDatabase {
|
||||
if (!this.db) throw new DatabaseError('AriaEngine storage not opened', 'ARIA_DB_NOT_OPEN');
|
||||
return this.db;
|
||||
}
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// Memory Backend（回退 / 测试用）
|
||||
// =======================================================================
|
||||
|
||||
export class MemoryBackend implements IStorageBackend {
|
||||
private store: Map<string, ArrayBuffer> = new Map();
|
||||
private opened = false;
|
||||
|
||||
async open(_name: string): Promise<void> {
|
||||
this.opened = true;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.store.clear();
|
||||
this.opened = false;
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.opened;
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
return this.store.get(key) ?? null;
|
||||
}
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
this.store.set(key, data);
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
this.store.delete(key);
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
return Array.from(this.store.keys());
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
return this.store.has(key);
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
this.store.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* AriaEngine File Manager — 页面文件管理 + PageIO 实现
|
||||
* @module engine/aria/store/file_manager
|
||||
*
|
||||
* 负责管理页面文件的生命周期：分配/释放页面 ID，读写页面。
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from './backend';
|
||||
import type { PageIO } from '../buffer/pool';
|
||||
import { PAGE_SIZE, PageType } from '../types';
|
||||
import { initPageHeader } from '../page/header';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileManager (implements PageIO)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class FileManager implements PageIO {
|
||||
private backend: IStorageBackend;
|
||||
private nextPageId = 0;
|
||||
private metaLoaded = false;
|
||||
private dbName = '';
|
||||
|
||||
constructor(backend: IStorageBackend) {
|
||||
this.backend = backend;
|
||||
}
|
||||
|
||||
/** 初始化：从存储中读取元数据 */
|
||||
async init(dbName: string): Promise<void> {
|
||||
this.dbName = dbName;
|
||||
// 读取 nextPageId
|
||||
const meta = await this.backend.read('__aria_meta');
|
||||
if (meta) {
|
||||
const view = new DataView(meta);
|
||||
this.nextPageId = view.getUint32(0, false);
|
||||
} else {
|
||||
this.nextPageId = 1;
|
||||
await this.saveMeta();
|
||||
}
|
||||
this.metaLoaded = true;
|
||||
}
|
||||
|
||||
// ---- PageIO ----
|
||||
|
||||
async readPage(pageId: number): Promise<ArrayBuffer | null> {
|
||||
const key = `pg_${pageId}`;
|
||||
const data = await this.backend.read(key);
|
||||
if (!data) {
|
||||
// 第一次访问：创建新页面
|
||||
return this.createEmptyPage(pageId, PageType.DATA);
|
||||
}
|
||||
|
||||
// 确保大小正确
|
||||
if (data.byteLength < PAGE_SIZE) {
|
||||
const padded = new ArrayBuffer(PAGE_SIZE);
|
||||
new Uint8Array(padded).set(new Uint8Array(data));
|
||||
return padded;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
async writePage(pageId: number, data: ArrayBuffer): Promise<void> {
|
||||
const key = `pg_${pageId}`;
|
||||
await this.backend.write(key, data);
|
||||
}
|
||||
|
||||
async allocatePageId(): Promise<number> {
|
||||
const id = this.nextPageId++;
|
||||
await this.saveMeta();
|
||||
return id;
|
||||
}
|
||||
|
||||
async freePageId(_pageId: number): Promise<void> {
|
||||
// 简化实现：不回收 pageId
|
||||
const key = `pg_${_pageId}`;
|
||||
await this.backend.delete(key);
|
||||
}
|
||||
|
||||
// ---- 表页面分配 ----
|
||||
|
||||
/**
|
||||
* 分配一个新的表元数据页面。
|
||||
*/
|
||||
async allocateTableRootPage(): Promise<number> {
|
||||
const pageId = await this.allocatePageId();
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, PageType.META);
|
||||
await this.writePage(pageId, data);
|
||||
return pageId;
|
||||
}
|
||||
|
||||
// ---- 辅助 ----
|
||||
|
||||
private async saveMeta(): Promise<void> {
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setUint32(0, this.nextPageId, false);
|
||||
await this.backend.write('__aria_meta', buf);
|
||||
}
|
||||
|
||||
private createEmptyPage(pageId: number, type: PageType): ArrayBuffer {
|
||||
const buf = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(buf, pageId, type);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/** 清空所有数据 */
|
||||
async clearAll(): Promise<void> {
|
||||
await this.backend.clear();
|
||||
this.nextPageId = 1;
|
||||
await this.saveMeta();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* AriaEngine MVCC — 多版本并发控制
|
||||
* @module engine/aria/transaction/mvcc
|
||||
*
|
||||
* 实现快照隔离 (Snapshot Isolation)。
|
||||
* 每个事务看到数据库在事务开始时的快照。
|
||||
*/
|
||||
|
||||
import type { RowVersion, TxnEntry } from '../types';
|
||||
import { TransactionState } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MVCCManager
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class MVCCManager {
|
||||
/** 所有行版本的存储：tableName.key → 版本链 */
|
||||
private versionStore: Map<string, RowVersion[]> = new Map();
|
||||
|
||||
/** 活跃事务表：txnId → TxnEntry */
|
||||
private activeTxns: Map<number, TxnEntry> = new Map();
|
||||
|
||||
/** 事务 ID 计数器 */
|
||||
private nextTxnId = 1;
|
||||
|
||||
/** 全局提交序列号（用于可见性判断） */
|
||||
private globalCommitLsn = 0;
|
||||
|
||||
// =======================================================================
|
||||
// 事务管理
|
||||
// =======================================================================
|
||||
|
||||
/** 开始一个事务，返回事务 ID */
|
||||
beginTransaction(): number {
|
||||
const txnId = this.nextTxnId++;
|
||||
this.activeTxns.set(txnId, {
|
||||
txnId,
|
||||
state: TransactionState.ACTIVE,
|
||||
snapshotLsn: this.globalCommitLsn,
|
||||
startTime: Date.now(),
|
||||
});
|
||||
return txnId;
|
||||
}
|
||||
|
||||
/** 提交事务 */
|
||||
commitTransaction(txnId: number): void {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) throw new Error(`Transaction ${txnId} not found`);
|
||||
|
||||
txn.state = TransactionState.COMMITTED;
|
||||
this.globalCommitLsn++;
|
||||
|
||||
// 标记此事务写入的所有版本为已提交
|
||||
for (const [, versions] of this.versionStore) {
|
||||
for (const version of versions) {
|
||||
if (version.txnId === txnId) {
|
||||
version.committed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理已提交事务的记录
|
||||
this.activeTxns.delete(txnId);
|
||||
}
|
||||
|
||||
/** 回滚事务 */
|
||||
rollbackTransaction(txnId: number): void {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) throw new Error(`Transaction ${txnId} not found`);
|
||||
|
||||
txn.state = TransactionState.ABORTED;
|
||||
|
||||
// 移除此事务写入的所有版本
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||
if (filtered.length === 0) {
|
||||
this.versionStore.delete(tableKey);
|
||||
} else {
|
||||
this.versionStore.set(tableKey, filtered);
|
||||
}
|
||||
}
|
||||
|
||||
this.activeTxns.delete(txnId);
|
||||
}
|
||||
|
||||
/** 检查事务是否活跃 */
|
||||
isActive(txnId: number): boolean {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
return txn !== undefined && txn.state === TransactionState.ACTIVE;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 版本读写
|
||||
// =======================================================================
|
||||
|
||||
/**
|
||||
* 写入一行（创建新版本）。
|
||||
*/
|
||||
writeVersion(
|
||||
tableName: string,
|
||||
key: string,
|
||||
data: Record<string, unknown>,
|
||||
txnId: number,
|
||||
): void {
|
||||
const tableKey = `${tableName}.${key}`;
|
||||
const versions = this.versionStore.get(tableKey) ?? [];
|
||||
|
||||
const newVersion: RowVersion = {
|
||||
txnId,
|
||||
data,
|
||||
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
|
||||
committed: false,
|
||||
};
|
||||
|
||||
versions.push(newVersion);
|
||||
this.versionStore.set(tableKey, versions);
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取一行（对指定事务可见的最新版本）。
|
||||
*/
|
||||
readVersion(
|
||||
tableName: string,
|
||||
key: string,
|
||||
txnId: number,
|
||||
): Record<string, unknown> | null {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) return null;
|
||||
|
||||
const tableKey = `${tableName}.${key}`;
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions || versions.length === 0) return null;
|
||||
|
||||
// 从最新版本向前遍历
|
||||
for (let i = versions.length - 1; i >= 0; i--) {
|
||||
const version = versions[i];
|
||||
|
||||
// 1. 如果是当前事务写入的（未提交），可见
|
||||
if (version.txnId === txnId) {
|
||||
return version.data;
|
||||
}
|
||||
|
||||
// 2. 如果是已提交的版本，且在快照 LSN 之前提交，可见
|
||||
if (version.committed) {
|
||||
// 简化：所有已提交版本都可见
|
||||
return version.data;
|
||||
}
|
||||
|
||||
// 3. 其他事务的未提交版本，不可见，继续找更早的版本
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除一行（创建墓碑版本）。
|
||||
*/
|
||||
deleteVersion(tableName: string, key: string, txnId: number): void {
|
||||
this.writeVersion(tableName, key, { __mvcc_tombstone: true } as unknown as Record<string, unknown>, txnId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有行的最新已提交版本（用于非事务读取）。
|
||||
*/
|
||||
getLatestCommittedVersions(
|
||||
tableName: string,
|
||||
): Record<string, Record<string, unknown>> {
|
||||
const result: Record<string, Record<string, unknown>> = {};
|
||||
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (!tableKey.startsWith(`${tableName}.`)) continue;
|
||||
const key = tableKey.slice(tableName.length + 1);
|
||||
|
||||
for (let i = versions.length - 1; i >= 0; i--) {
|
||||
const version = versions[i];
|
||||
if (version.committed && !(version.data as unknown as Record<string, unknown>).__mvcc_tombstone) {
|
||||
result[key] = version.data;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理过旧版本（GC）。
|
||||
* 保留每个 key 的最新 N 个已提交版本。
|
||||
*/
|
||||
gc(maxVersionsPerKey: number = 100): void {
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (versions.length <= maxVersionsPerKey) continue;
|
||||
|
||||
// 保留最新的 maxVersionsPerKey 个版本
|
||||
const pruned = versions.slice(versions.length - maxVersionsPerKey);
|
||||
this.versionStore.set(tableKey, pruned);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有未提交事务中的 key 列表。
|
||||
*/
|
||||
getActiveWriteKeys(tableName: string, txnId: number): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
const prefix = `${tableName}.`;
|
||||
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (!tableKey.startsWith(prefix)) continue;
|
||||
const latestVersion = versions[versions.length - 1];
|
||||
if (latestVersion.txnId === txnId && !latestVersion.committed) {
|
||||
keys.add(tableKey.slice(prefix.length));
|
||||
}
|
||||
}
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理指定表的所有版本。
|
||||
*/
|
||||
clearTable(tableName: string): void {
|
||||
const prefix = `${tableName}.`;
|
||||
for (const [tableKey] of this.versionStore) {
|
||||
if (tableKey.startsWith(prefix)) {
|
||||
this.versionStore.delete(tableKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取活跃事务数。
|
||||
*/
|
||||
getActiveTxnCount(): number {
|
||||
return this.activeTxns.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取全局 LSN。
|
||||
*/
|
||||
getGlobalLSN(): number {
|
||||
return this.globalCommitLsn;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
/**
|
||||
* AriaEngine Types — 内部类型定义
|
||||
* @module engine/aria/types
|
||||
*
|
||||
* 页面式存储引擎的所有内部枚举、接口和常量。
|
||||
*/
|
||||
|
||||
// =============================================================================
|
||||
// 页面常量
|
||||
// =============================================================================
|
||||
|
||||
/** 页面大小：4KB */
|
||||
export const PAGE_SIZE = 4096;
|
||||
|
||||
/** 页面头大小：16 字节 */
|
||||
export const PAGE_HEADER_SIZE = 16;
|
||||
|
||||
/** 每个 Slot 目录项大小：4 字节 (offset: u16 + len: u16) */
|
||||
export const SLOT_ENTRY_SIZE = 4;
|
||||
|
||||
/** 页面数据区起始偏移（头部之后） */
|
||||
export const PAGE_DATA_START = PAGE_HEADER_SIZE;
|
||||
|
||||
/** 无效页面 ID */
|
||||
export const INVALID_PAGE_ID = 0xFFFFFFFF;
|
||||
|
||||
// =============================================================================
|
||||
// 页面类型
|
||||
// =============================================================================
|
||||
|
||||
export enum PageType {
|
||||
/** 数据页面：存储行数据 */
|
||||
DATA = 1,
|
||||
/** 索引页面：存储索引节点 */
|
||||
INDEX = 2,
|
||||
/** 溢出页面：存储大字段 */
|
||||
OVERFLOW = 3,
|
||||
/** 元数据页面：存储表/库元信息 */
|
||||
META = 4,
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 页面头部（16 字节）
|
||||
// =============================================================================
|
||||
|
||||
export interface PageHeader {
|
||||
/** 页面 ID（全局唯一） */
|
||||
pageId: number;
|
||||
/** 页面类型 */
|
||||
type: PageType;
|
||||
/** 空闲空间起始偏移（slot 区结束位置） */
|
||||
freeStart: number;
|
||||
/** 数据区结束偏移（从页面底部向上增长） */
|
||||
freeEnd: number;
|
||||
/** 当前 slot 数量 */
|
||||
slotCount: number;
|
||||
/** CRC32 校验和 */
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Slot 目录项（4 字节）
|
||||
// =============================================================================
|
||||
|
||||
export interface SlotEntry {
|
||||
/** 行数据在页面内的偏移 */
|
||||
offset: number;
|
||||
/** 行数据长度（字节） */
|
||||
length: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 页面句柄（Buffer Pool 中的页面）
|
||||
// =============================================================================
|
||||
|
||||
export interface PageHandle {
|
||||
/** 页面 ID */
|
||||
pageId: number;
|
||||
/** 页面类型 */
|
||||
type: PageType;
|
||||
/** 页面数据缓冲区（4KB ArrayBuffer） */
|
||||
data: ArrayBuffer;
|
||||
/** 是否被修改（脏页） */
|
||||
dirty: boolean;
|
||||
/** 引用计数（pin count） */
|
||||
pins: number;
|
||||
/** LRU 链表前驱 */
|
||||
prev: PageHandle | null;
|
||||
/** LRU 链表后继 */
|
||||
next: PageHandle | null;
|
||||
/** 最后访问时间戳 */
|
||||
lastAccess: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 行编解码
|
||||
// =============================================================================
|
||||
|
||||
/** 行/元组的二进制表示 */
|
||||
export interface SerializedTuple {
|
||||
/** 序列化后的字节数组 */
|
||||
bytes: Uint8Array;
|
||||
/** 该行中 null 列的位图 */
|
||||
nullBitmap: Uint8Array;
|
||||
}
|
||||
|
||||
/** 列类型（内部二进制编码用） */
|
||||
export enum ColumnEncoding {
|
||||
STRING = 1,
|
||||
NUMBER = 2,
|
||||
BOOLEAN = 3,
|
||||
DATE = 4,
|
||||
JSON = 5,
|
||||
NULL = 6,
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// LSM-Tree
|
||||
// =============================================================================
|
||||
|
||||
/** MemTable 最大大小（默认 4MB） */
|
||||
export const DEFAULT_MEMTABLE_SIZE = 4 * 1024 * 1024;
|
||||
|
||||
/** SSTable 中每个 Data Block 的默认大小 */
|
||||
export const DEFAULT_BLOCK_SIZE = 4096;
|
||||
|
||||
/** Bloom Filter 每 key 的默认位数 */
|
||||
export const DEFAULT_BLOOM_BITS_PER_KEY = 10;
|
||||
|
||||
/** SSTable 最大层级 */
|
||||
export const MAX_LSM_LEVELS = 7;
|
||||
|
||||
/** 每层之间的大小倍数 */
|
||||
export const DEFAULT_LEVEL_SIZE_MULTIPLIER = 10;
|
||||
|
||||
/** SSTable 元数据 */
|
||||
export interface SSTableMeta {
|
||||
/** SSTable 文件 ID */
|
||||
id: number;
|
||||
/** 所在层级 */
|
||||
level: number;
|
||||
/** 最小 key */
|
||||
minKey: string;
|
||||
/** 最大 key */
|
||||
maxKey: string;
|
||||
/** 数据块数量 */
|
||||
blockCount: number;
|
||||
/** 总大小（字节） */
|
||||
totalSize: number;
|
||||
/** Bloom Filter 序列化数据 */
|
||||
bloomData: Uint8Array | null;
|
||||
}
|
||||
|
||||
/** SSTable 内部的 Data Block */
|
||||
export interface DataBlock {
|
||||
/** 该块内的 key-value 条目数 */
|
||||
entryCount: number;
|
||||
/** 该块数据区 */
|
||||
data: Uint8Array;
|
||||
/** 该块起始 key */
|
||||
startKey: string;
|
||||
/** 该块结束 key */
|
||||
endKey: string;
|
||||
}
|
||||
|
||||
/** 索引块条目：key → data block offset */
|
||||
export interface IndexEntry {
|
||||
/** 到此 block 的最后一个 key */
|
||||
key: string;
|
||||
/** data block 在 SSTable 文件中的偏移 */
|
||||
blockOffset: number;
|
||||
/** data block 大小 */
|
||||
blockSize: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// WAL (Write-Ahead Log)
|
||||
// =============================================================================
|
||||
|
||||
/** WAL 记录类型 */
|
||||
export enum WALRecordType {
|
||||
INSERT = 1,
|
||||
UPDATE = 2,
|
||||
DELETE = 3,
|
||||
BEGIN = 4,
|
||||
COMMIT = 5,
|
||||
ROLLBACK = 6,
|
||||
CREATE_TABLE = 7,
|
||||
DROP_TABLE = 8,
|
||||
}
|
||||
|
||||
/** 单条 WAL 记录 */
|
||||
export interface WALRecord {
|
||||
/** 日志序列号 */
|
||||
lsn: number;
|
||||
/** 记录类型 */
|
||||
type: WALRecordType;
|
||||
/** 事务 ID */
|
||||
txnId: number;
|
||||
/** 表名 */
|
||||
tableName: string;
|
||||
/** 主键值 */
|
||||
key: string;
|
||||
/** 操作数据（INSERT/UPDATE 时有效） */
|
||||
data?: Record<string, unknown>;
|
||||
/** 校验和 */
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// MVCC
|
||||
// =============================================================================
|
||||
|
||||
/** 事务隔离级别 */
|
||||
export enum IsolationLevel {
|
||||
READ_COMMITTED = 1,
|
||||
SNAPSHOT = 2,
|
||||
}
|
||||
|
||||
/** 事务状态 */
|
||||
export enum TransactionState {
|
||||
ACTIVE = 1,
|
||||
COMMITTED = 2,
|
||||
ABORTED = 3,
|
||||
}
|
||||
|
||||
/** 行版本 */
|
||||
export interface RowVersion {
|
||||
/** 事务 ID（创建此版本的事务） */
|
||||
txnId: number;
|
||||
/** 版本数据 */
|
||||
data: Record<string, unknown>;
|
||||
/** 指向上一版本的指针（undo 链） */
|
||||
prevVersion: RowVersion | null;
|
||||
/** 该版本是否已提交 */
|
||||
committed: boolean;
|
||||
}
|
||||
|
||||
/** 活跃事务表项 */
|
||||
export interface TxnEntry {
|
||||
/** 事务 ID */
|
||||
txnId: number;
|
||||
/** 事务状态 */
|
||||
state: TransactionState;
|
||||
/** 快照序列号（用于可见性判断） */
|
||||
snapshotLsn: number;
|
||||
/** 事务开始时间 */
|
||||
startTime: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Buffer Pool
|
||||
// =============================================================================
|
||||
|
||||
/** Buffer Pool 默认容量：256 页 ≈ 1MB */
|
||||
export const DEFAULT_BUFFER_POOL_PAGES = 256;
|
||||
|
||||
// =============================================================================
|
||||
// AriaEngine 配置
|
||||
// =============================================================================
|
||||
|
||||
export interface AriaEngineConfig {
|
||||
/** 页面大小（默认 4096） */
|
||||
pageSize?: number;
|
||||
/** Buffer Pool 页面数量（默认 256） */
|
||||
bufferPoolPages?: number;
|
||||
/** MemTable 刷盘阈值（默认 4MB） */
|
||||
memtableSizeThreshold?: number;
|
||||
/** LSM 层级之间的容量倍数（默认 10） */
|
||||
levelSizeMultiplier?: number;
|
||||
/** Bloom Filter 每 key 位数（默认 10） */
|
||||
bloomFilterBitsPerKey?: number;
|
||||
/** 是否启用 WAL（默认 true） */
|
||||
walEnabled?: boolean;
|
||||
/** WAL 同步模式 */
|
||||
walSyncMode?: 'full' | 'batch' | 'none';
|
||||
/** Checkpoint 间隔（操作数，默认 1000） */
|
||||
checkpointInterval?: number;
|
||||
/** 是否启用页面压缩（默认 false） */
|
||||
compression?: boolean;
|
||||
/** 存储后端 */
|
||||
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
||||
}
|
||||
|
||||
export const DEFAULT_ARIA_CONFIG: Required<AriaEngineConfig> = {
|
||||
pageSize: PAGE_SIZE,
|
||||
bufferPoolPages: DEFAULT_BUFFER_POOL_PAGES,
|
||||
memtableSizeThreshold: DEFAULT_MEMTABLE_SIZE,
|
||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||
walEnabled: true,
|
||||
walSyncMode: 'batch',
|
||||
checkpointInterval: 1000,
|
||||
compression: false,
|
||||
storageBackend: 'indexeddb',
|
||||
};
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* AriaEngine Checkpoint — 检查点机制
|
||||
* @module engine/aria/wal/checkpoint
|
||||
*/
|
||||
|
||||
import type { LSM } from '../index/lsm';
|
||||
import type { WAL } from './log';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 简化的 flush 接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface Flushable {
|
||||
flushAll(): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CheckpointManager
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class CheckpointManager {
|
||||
private lsm: LSM;
|
||||
private wal: WAL;
|
||||
private flushable: Flushable | null;
|
||||
private interval: number;
|
||||
private opCount = 0;
|
||||
|
||||
constructor(
|
||||
lsm: LSM,
|
||||
wal: WAL,
|
||||
flushable: Flushable | null = null,
|
||||
interval: number = 1000,
|
||||
) {
|
||||
this.lsm = lsm;
|
||||
this.wal = wal;
|
||||
this.flushable = flushable;
|
||||
this.interval = interval;
|
||||
}
|
||||
|
||||
async tick(): Promise<void> {
|
||||
this.opCount++;
|
||||
if (this.opCount >= this.interval) {
|
||||
await this.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
async checkpoint(): Promise<void> {
|
||||
await this.lsm.flush();
|
||||
if (this.flushable) {
|
||||
await this.flushable.flushAll();
|
||||
}
|
||||
await this.wal.checkpoint();
|
||||
this.opCount = 0;
|
||||
}
|
||||
|
||||
async forceCheckpoint(): Promise<void> {
|
||||
await this.checkpoint();
|
||||
}
|
||||
|
||||
setInterval(ops: number): void {
|
||||
this.interval = ops;
|
||||
}
|
||||
|
||||
getOpCount(): number {
|
||||
return this.opCount;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* AriaEngine WAL — Write-Ahead Log
|
||||
* @module engine/aria/wal/log
|
||||
*
|
||||
* 崩溃恢复前的写操作持久化日志。
|
||||
*
|
||||
* WAL 文件格式:
|
||||
* ┌──────────┬──────────────┬──────────┐
|
||||
* │ Record 1│ Record 2 │ ... │
|
||||
* │ 4B LSN │ │ │
|
||||
* │ 1B type │ │ │
|
||||
* │ 4B txnId│ │ │
|
||||
* │ 2B tblLen│ │ │
|
||||
* │ N table│ │ │
|
||||
* │ 2B keyLen│ │ │
|
||||
* │ N key │ │ │
|
||||
* │ 4B jsonLen│ │ │
|
||||
* │ N json │ │ │
|
||||
* │ 4B CRC │ │ │
|
||||
* └──────────┴──────────────┴──────────┘
|
||||
*/
|
||||
|
||||
import { WALRecordType, type WALRecord } from '../types';
|
||||
import type { BufferPool } from '../buffer/pool';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL 存储接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WALStore {
|
||||
/** 追加 WAL 记录 */
|
||||
append(data: Uint8Array): Promise<void>;
|
||||
/** 读取所有 WAL 记录 */
|
||||
readAll(): Promise<Uint8Array>;
|
||||
/** 截断 WAL（checkpoint 后清理） */
|
||||
truncate(): Promise<void>;
|
||||
/** 检查 WAL 是否存在 */
|
||||
exists(): Promise<boolean>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class WAL {
|
||||
private lsn = 0;
|
||||
private store: WALStore;
|
||||
private enabled: boolean;
|
||||
private buffer: Uint8Array[] = [];
|
||||
private syncMode: 'full' | 'batch' | 'none';
|
||||
|
||||
constructor(store: WALStore, enabled: boolean = true, syncMode: 'full' | 'batch' | 'none' = 'batch') {
|
||||
this.store = store;
|
||||
this.enabled = enabled;
|
||||
this.syncMode = syncMode;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 写入
|
||||
// =======================================================================
|
||||
|
||||
/** 追加一条 WAL 记录 */
|
||||
append(record: Omit<WALRecord, 'lsn' | 'checksum'>): void {
|
||||
if (!this.enabled) return;
|
||||
|
||||
this.lsn++;
|
||||
const fullRecord: WALRecord = {
|
||||
...record,
|
||||
lsn: this.lsn,
|
||||
checksum: 0, // 稍后计算
|
||||
};
|
||||
|
||||
const bytes = this.encodeRecord(fullRecord);
|
||||
|
||||
if (this.syncMode === 'full') {
|
||||
this.store.append(bytes).catch(() => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn('[AriaEngine WAL] Failed to append record');
|
||||
});
|
||||
} else if (this.syncMode === 'batch') {
|
||||
this.buffer.push(bytes);
|
||||
}
|
||||
// 'none' mode: 不写 WAL
|
||||
}
|
||||
|
||||
/** 批量刷新缓冲的 WAL 记录 */
|
||||
async flush(): Promise<void> {
|
||||
if (!this.enabled || this.buffer.length === 0) return;
|
||||
|
||||
const totalLen = this.buffer.reduce((sum, b) => sum + b.byteLength, 0);
|
||||
const combined = new Uint8Array(totalLen);
|
||||
let offset = 0;
|
||||
for (const buf of this.buffer) {
|
||||
combined.set(buf, offset);
|
||||
offset += buf.byteLength;
|
||||
}
|
||||
|
||||
await this.store.append(combined);
|
||||
this.buffer = [];
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 恢复
|
||||
// =======================================================================
|
||||
|
||||
/** 从 WAL 恢复未提交的事务数据 */
|
||||
async recover(
|
||||
applyRecord: (record: WALRecord) => void,
|
||||
): Promise<number> {
|
||||
if (!this.enabled) return 0;
|
||||
|
||||
const exists = await this.store.exists();
|
||||
if (!exists) return 0;
|
||||
|
||||
const data = await this.store.readAll();
|
||||
if (data.byteLength === 0) return 0;
|
||||
|
||||
const records = this.decodeAllRecords(data);
|
||||
for (const record of records) {
|
||||
applyRecord(record);
|
||||
}
|
||||
|
||||
this.lsn = records.length > 0 ? records[records.length - 1].lsn : 0;
|
||||
return records.length;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// Checkpoint
|
||||
// =======================================================================
|
||||
|
||||
/** Checkpoint 后清空 WAL */
|
||||
async checkpoint(): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
await this.flush();
|
||||
await this.store.truncate();
|
||||
this.lsn = 0;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 统计
|
||||
// =======================================================================
|
||||
|
||||
isEnabled(): boolean {
|
||||
return this.enabled;
|
||||
}
|
||||
|
||||
getLSN(): number {
|
||||
return this.lsn;
|
||||
}
|
||||
|
||||
getBufferedCount(): number {
|
||||
return this.buffer.length;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 编解码
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private encodeRecord(record: WALRecord): Uint8Array {
|
||||
const encoder = new TextEncoder();
|
||||
const tableBytes = encoder.encode(record.tableName);
|
||||
const keyBytes = encoder.encode(record.key);
|
||||
const jsonStr = record.data ? JSON.stringify(record.data) : '';
|
||||
const jsonBytes = encoder.encode(jsonStr);
|
||||
|
||||
const size =
|
||||
4 + // LSN
|
||||
1 + // type
|
||||
4 + // txnId
|
||||
2 + tableBytes.length + // table
|
||||
2 + keyBytes.length + // key
|
||||
4 + jsonBytes.length + // json
|
||||
4; // CRC
|
||||
|
||||
const buf = new ArrayBuffer(size);
|
||||
const view = new DataView(buf);
|
||||
let offset = 0;
|
||||
|
||||
view.setUint32(offset, record.lsn, false);
|
||||
offset += 4;
|
||||
view.setUint8(offset, record.type);
|
||||
offset += 1;
|
||||
view.setUint32(offset, record.txnId, false);
|
||||
offset += 4;
|
||||
|
||||
view.setUint16(offset, tableBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(buf).set(tableBytes, offset);
|
||||
offset += tableBytes.length;
|
||||
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(buf).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
|
||||
view.setUint32(offset, jsonBytes.length, false);
|
||||
offset += 4;
|
||||
new Uint8Array(buf).set(jsonBytes, offset);
|
||||
offset += jsonBytes.length;
|
||||
|
||||
// 简单 CRC
|
||||
let crc = 0;
|
||||
const u8 = new Uint8Array(buf, 0, offset);
|
||||
for (let i = 0; i < u8.length; i++) {
|
||||
crc = ((crc << 5) - crc + u8[i]) | 0;
|
||||
}
|
||||
view.setUint32(offset, crc >>> 0, false);
|
||||
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
|
||||
private decodeAllRecords(data: Uint8Array): WALRecord[] {
|
||||
const records: WALRecord[] = [];
|
||||
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
while (offset + 15 <= data.byteLength) {
|
||||
try {
|
||||
const lsn = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
const type = view.getUint8(offset) as WALRecordType;
|
||||
offset += 1;
|
||||
const txnId = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
|
||||
const tableLen = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const tableName = new TextDecoder().decode(data.slice(offset, offset + tableLen));
|
||||
offset += tableLen;
|
||||
|
||||
const keyLen = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
const key = new TextDecoder().decode(data.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
|
||||
const jsonLen = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
let recordData: Record<string, unknown> | undefined;
|
||||
if (jsonLen > 0) {
|
||||
const json = new TextDecoder().decode(data.slice(offset, offset + jsonLen));
|
||||
try {
|
||||
recordData = JSON.parse(json);
|
||||
} catch { /* ok */ }
|
||||
}
|
||||
offset += jsonLen;
|
||||
|
||||
// 跳过 CRC
|
||||
offset += 4;
|
||||
|
||||
records.push({
|
||||
lsn,
|
||||
type,
|
||||
txnId,
|
||||
tableName,
|
||||
key,
|
||||
data: recordData,
|
||||
checksum: 0,
|
||||
});
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
}
|
||||
@@ -7,3 +7,5 @@ export type { IStorageEngine } from './interface';
|
||||
export { MemoryEngine } from './memory';
|
||||
export { IndexedDBEngine } from './indexeddb';
|
||||
export { OPFSEngine } from './opfs';
|
||||
export { AriaEngine } from './aria/index';
|
||||
export type { AriaEngineConfig } from './aria/types';
|
||||
|
||||
@@ -80,7 +80,11 @@ export type { Statement, SelectStatement, InsertStatement, UpdateStatement, Dele
|
||||
export { MemoryEngine } from './engine/memory';
|
||||
export { IndexedDBEngine } from './engine/indexeddb';
|
||||
export { OPFSEngine } from './engine/opfs';
|
||||
export { AriaEngine } from './engine/aria/index';
|
||||
export { HybridEngine } from './hybrid/index';
|
||||
export { Table } from './table/table';
|
||||
export { parse } from './sql/parser';
|
||||
export { tokenize } from './sql/lexer';
|
||||
|
||||
// AriaEngine 类型
|
||||
export type { AriaEngineConfig } from './engine/aria/types';
|
||||
|
||||
Reference in New Issue
Block a user