release: v0.5.1 — 存储后端生产级硬化（CRC-32/全库加密/WAL分片/页面化存储/多标签页锁/e2e）+ 深度审查修复（假实现接线/死代码清理）
This commit is contained in:
thzxx
+177
-206
@@ -1,206 +1,177 @@
|
||||
/**
|
||||
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
|
||||
* @module engine/aria/buffer/eviction
|
||||
*/
|
||||
|
||||
import type { PageHandle } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU 双向链表
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* LRU 链表管理器 — 双向链表，头部是 most recently used，尾部是 least recently used。
|
||||
*/
|
||||
export class LRUList {
|
||||
private head: PageHandle | null = null;
|
||||
private tail: PageHandle | null = null;
|
||||
private _size = 0;
|
||||
|
||||
get size(): number {
|
||||
return this._size;
|
||||
}
|
||||
|
||||
/** 将页面移到链表头部。如果是新页面则插入，已存在则移动。 */
|
||||
moveToHead(page: PageHandle): void {
|
||||
// 如果已经在头部，无需操作
|
||||
if (this.head === page) return;
|
||||
|
||||
// 检测是否在链表中
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
|
||||
if (inList) {
|
||||
// 先从当前位置移除
|
||||
this.detach(page);
|
||||
} else {
|
||||
this._size++;
|
||||
}
|
||||
|
||||
// 插入头部
|
||||
page.prev = null;
|
||||
page.next = this.head;
|
||||
if (this.head) {
|
||||
this.head.prev = page;
|
||||
}
|
||||
this.head = page;
|
||||
if (!this.tail) {
|
||||
this.tail = page;
|
||||
}
|
||||
}
|
||||
|
||||
/** 从链表中移除页面 */
|
||||
remove(page: PageHandle): void {
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (!inList) return;
|
||||
|
||||
this.detach(page);
|
||||
this._size = Math.max(0, this._size - 1);
|
||||
}
|
||||
|
||||
/** 内部：只调整指针，不修改 _size */
|
||||
private detach(page: PageHandle): void {
|
||||
if (page.prev) {
|
||||
page.prev.next = page.next;
|
||||
} else if (this.head === page) {
|
||||
this.head = page.next;
|
||||
}
|
||||
|
||||
if (page.next) {
|
||||
page.next.prev = page.prev;
|
||||
} else if (this.tail === page) {
|
||||
this.tail = page.prev;
|
||||
}
|
||||
|
||||
page.prev = null;
|
||||
page.next = null;
|
||||
}
|
||||
|
||||
/** 获取 LRU 尾部（最久未使用的页面） */
|
||||
getLRU(): PageHandle | null {
|
||||
return this.tail;
|
||||
}
|
||||
|
||||
/** 弹出 LRU 尾部 */
|
||||
popLRU(): PageHandle | null {
|
||||
const lru = this.tail;
|
||||
if (lru) {
|
||||
this.remove(lru);
|
||||
}
|
||||
return lru;
|
||||
}
|
||||
|
||||
/** 清空链表 */
|
||||
clear(): void {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
|
||||
/** 获取所有页面（用于迭代） */
|
||||
getAllPages(): PageHandle[] {
|
||||
const pages: PageHandle[] = [];
|
||||
let current = this.head;
|
||||
while (current) {
|
||||
pages.push(current);
|
||||
current = current.next;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Eviction 策略
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type EvictionCallback = (page: PageHandle) => Promise<void>;
|
||||
|
||||
/**
|
||||
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
|
||||
*/
|
||||
export class EvictionManager {
|
||||
private lru: LRUList;
|
||||
private onEvict: EvictionCallback;
|
||||
private capacity: number;
|
||||
|
||||
constructor(capacity: number, onEvict: EvictionCallback) {
|
||||
this.lru = new LRUList();
|
||||
this.capacity = capacity;
|
||||
this.onEvict = onEvict;
|
||||
}
|
||||
|
||||
/** 访问页面，更新 LRU */
|
||||
access(page: PageHandle): void {
|
||||
page.lastAccess = Date.now();
|
||||
this.lru.moveToHead(page);
|
||||
}
|
||||
|
||||
/** 添加新页面到池中 */
|
||||
add(page: PageHandle): void {
|
||||
this.access(page);
|
||||
}
|
||||
|
||||
/** 移除指定页面 */
|
||||
remove(page: PageHandle): void {
|
||||
this.lru.remove(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* 驱逐页面直到池中有足够空间。
|
||||
* 只驱逐未 pin 的干净页面（dirty=false）。
|
||||
* 如果没有干净页面可驱逐，尝试刷脏页。
|
||||
*/
|
||||
async evictIfNeeded(count: number): Promise<number> {
|
||||
let evicted = 0;
|
||||
|
||||
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
|
||||
// 找到可驱逐的页面
|
||||
const victim = this.findEvictionCandidate();
|
||||
if (!victim) break;
|
||||
|
||||
// 脏页先刷盘
|
||||
if (victim.dirty) {
|
||||
await this.onEvict(victim);
|
||||
victim.dirty = false;
|
||||
}
|
||||
|
||||
this.lru.remove(victim);
|
||||
evicted++;
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
/** 查找驱逐候选（优先干净页面，然后最久未用的脏页） */
|
||||
private findEvictionCandidate(): PageHandle | null {
|
||||
// 先从尾部找未 pin 的干净页面
|
||||
let current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0 && !current.dirty) return current;
|
||||
current = current.prev;
|
||||
}
|
||||
|
||||
// 没有干净页，找未 pin 的脏页
|
||||
current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0) return current;
|
||||
current = current.prev;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 获取当前大小 */
|
||||
getSize(): number {
|
||||
return this.lru.size;
|
||||
}
|
||||
|
||||
/** 获取容量 */
|
||||
getCapacity(): number {
|
||||
return this.capacity;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.lru.clear();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine Buffer Pool Eviction — LRU 驱逐策略
|
||||
* @module engine/aria/buffer/eviction
|
||||
*/
|
||||
|
||||
import type { PageHandle } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU 双向链表
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* LRU 链表管理器 — 双向链表，头部是 most recently used，尾部是 least recently used。
|
||||
*/
|
||||
export class LRUList {
|
||||
private head: PageHandle | null = null;
|
||||
private tail: PageHandle | null = null;
|
||||
private _size = 0;
|
||||
|
||||
get size(): number {
|
||||
return this._size;
|
||||
}
|
||||
|
||||
/** 将页面移到链表头部。如果是新页面则插入，已存在则移动。 */
|
||||
moveToHead(page: PageHandle): void {
|
||||
// 如果已经在头部，无需操作
|
||||
if (this.head === page) return;
|
||||
|
||||
// 检测是否在链表中
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
|
||||
if (inList) {
|
||||
// 先从当前位置移除
|
||||
this.detach(page);
|
||||
} else {
|
||||
this._size++;
|
||||
}
|
||||
|
||||
// 插入头部
|
||||
page.prev = null;
|
||||
page.next = this.head;
|
||||
if (this.head) {
|
||||
this.head.prev = page;
|
||||
}
|
||||
this.head = page;
|
||||
if (!this.tail) {
|
||||
this.tail = page;
|
||||
}
|
||||
}
|
||||
|
||||
/** 从链表中移除页面 */
|
||||
remove(page: PageHandle): void {
|
||||
const inList = page.prev !== null || page.next !== null || this.head === page || this.tail === page;
|
||||
if (!inList) return;
|
||||
|
||||
this.detach(page);
|
||||
this._size = Math.max(0, this._size - 1);
|
||||
}
|
||||
|
||||
/** 内部：只调整指针，不修改 _size */
|
||||
private detach(page: PageHandle): void {
|
||||
if (page.prev) {
|
||||
page.prev.next = page.next;
|
||||
} else if (this.head === page) {
|
||||
this.head = page.next;
|
||||
}
|
||||
|
||||
if (page.next) {
|
||||
page.next.prev = page.prev;
|
||||
} else if (this.tail === page) {
|
||||
this.tail = page.prev;
|
||||
}
|
||||
|
||||
page.prev = null;
|
||||
page.next = null;
|
||||
}
|
||||
|
||||
/** 清空链表 */
|
||||
clear(): void {
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
this._size = 0;
|
||||
}
|
||||
|
||||
/** 获取 LRU 尾部（最久未使用的页面） */
|
||||
getLRU(): PageHandle | null {
|
||||
return this.tail;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Eviction 策略
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type EvictionCallback = (page: PageHandle) => Promise<void>;
|
||||
|
||||
/**
|
||||
* 驱逐管理器 — 当 Buffer Pool 满时驱逐页面。
|
||||
*/
|
||||
export class EvictionManager {
|
||||
private lru: LRUList;
|
||||
private onEvict: EvictionCallback;
|
||||
private capacity: number;
|
||||
|
||||
constructor(capacity: number, onEvict: EvictionCallback) {
|
||||
this.lru = new LRUList();
|
||||
this.capacity = capacity;
|
||||
this.onEvict = onEvict;
|
||||
}
|
||||
|
||||
/** 访问页面，更新 LRU */
|
||||
access(page: PageHandle): void {
|
||||
page.lastAccess = Date.now();
|
||||
this.lru.moveToHead(page);
|
||||
}
|
||||
|
||||
/** 添加新页面到池中 */
|
||||
add(page: PageHandle): void {
|
||||
this.access(page);
|
||||
}
|
||||
|
||||
/** 移除指定页面 */
|
||||
remove(page: PageHandle): void {
|
||||
this.lru.remove(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* 驱逐页面直到池中有足够空间。
|
||||
* 只驱逐未 pin 的干净页面（dirty=false）。
|
||||
* 如果没有干净页面可驱逐，尝试刷脏页。
|
||||
*/
|
||||
async evictIfNeeded(count: number): Promise<number> {
|
||||
let evicted = 0;
|
||||
|
||||
while (this.lru.size + count > this.capacity && this.lru.size > 0) {
|
||||
// 找到可驱逐的页面
|
||||
const victim = this.findEvictionCandidate();
|
||||
if (!victim) break;
|
||||
|
||||
// 脏页先刷盘
|
||||
if (victim.dirty) {
|
||||
await this.onEvict(victim);
|
||||
victim.dirty = false;
|
||||
}
|
||||
|
||||
this.lru.remove(victim);
|
||||
evicted++;
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
/** 查找驱逐候选（优先干净页面，然后最久未用的脏页） */
|
||||
private findEvictionCandidate(): PageHandle | null {
|
||||
// 先从尾部找未 pin 的干净页面
|
||||
let current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0 && !current.dirty) return current;
|
||||
current = current.prev;
|
||||
}
|
||||
|
||||
// 没有干净页，找未 pin 的脏页
|
||||
current = this.lru.getLRU();
|
||||
while (current) {
|
||||
if (current.pins === 0) return current;
|
||||
current = current.prev;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.lru.clear();
|
||||
}
|
||||
}
|
||||
|
||||
+177
-185
@@ -1,185 +1,177 @@
|
||||
/**
|
||||
* AriaEngine Buffer Pool — 页面缓存池
|
||||
* @module engine/aria/buffer/pool
|
||||
*
|
||||
* 管理固定数量页面的 LRU 缓存，减少对底层储存的访问。
|
||||
*/
|
||||
|
||||
import type { PageHandle } from '../types';
|
||||
import { PageType, DEFAULT_BUFFER_POOL_PAGES } from '../types';
|
||||
import { createPage } from '../page/format';
|
||||
import { EvictionManager } from './eviction';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Page Read / Write 回调
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface PageIO {
|
||||
/** 从存储后端读取页面 */
|
||||
readPage(pageId: number): Promise<ArrayBuffer | null>;
|
||||
/** 将页面写入存储后端 */
|
||||
writePage(pageId: number, data: ArrayBuffer): Promise<void>;
|
||||
/** 分配新页面 ID */
|
||||
allocatePageId(): Promise<number>;
|
||||
/** 释放页面 ID */
|
||||
freePageId(pageId: number): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer Pool
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class BufferPool {
|
||||
private pages: Map<number, PageHandle> = new Map();
|
||||
private eviction: EvictionManager;
|
||||
private pageIO: PageIO;
|
||||
private nextPageId = 0;
|
||||
|
||||
constructor(pageIO: PageIO, capacity: number = DEFAULT_BUFFER_POOL_PAGES) {
|
||||
this.pageIO = pageIO;
|
||||
this.eviction = new EvictionManager(capacity, async (page) => {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 页面获取
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 获取页面（必要时从磁盘读取）。
|
||||
* 返回 pin 的页面，使用完成后必须调用 unpin()。
|
||||
*/
|
||||
async getPage(pageId: number): Promise<PageHandle | null> {
|
||||
// 已在池中
|
||||
let page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.access(page);
|
||||
page.pins++;
|
||||
return page;
|
||||
}
|
||||
|
||||
// 需要从磁盘加载
|
||||
const buffer = await this.pageIO.readPage(pageId);
|
||||
if (!buffer) return null;
|
||||
|
||||
// 确保有空间
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
|
||||
const type = new DataView(buffer).getUint8(4) as PageType;
|
||||
page = {
|
||||
pageId,
|
||||
type,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 1,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建新页面。
|
||||
*/
|
||||
async newPage(type: PageType = PageType.DATA): Promise<PageHandle> {
|
||||
const pageId = await this.pageIO.allocatePageId();
|
||||
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
|
||||
const page = createPage(pageId, type);
|
||||
page.pins = 1;
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* 释放页面的 pin。
|
||||
*/
|
||||
unpin(page: PageHandle): void {
|
||||
if (page.pins > 0) {
|
||||
page.pins--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记页面为脏（需要写回）。
|
||||
*/
|
||||
markDirty(page: PageHandle): void {
|
||||
page.dirty = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将脏页面刷新到磁盘。
|
||||
*/
|
||||
async flushPage(pageId: number): Promise<void> {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page && page.dirty) {
|
||||
await this.pageIO.writePage(pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新所有脏页面。
|
||||
*/
|
||||
async flushAll(): Promise<void> {
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从缓存中删除指定页面（不刷盘）。
|
||||
*/
|
||||
removePage(pageId: number): void {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.remove(page);
|
||||
this.pages.delete(pageId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空缓存池（先刷脏页）。
|
||||
*/
|
||||
async clear(): Promise<void> {
|
||||
await this.flushAll();
|
||||
this.pages.clear();
|
||||
this.eviction.clear();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 统计
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/** 获取当前缓存页面数 */
|
||||
getCachedPageCount(): number {
|
||||
return this.pages.size;
|
||||
}
|
||||
|
||||
/** 获取缓存容量 */
|
||||
getCapacity(): number {
|
||||
return this.eviction.getCapacity();
|
||||
}
|
||||
|
||||
/** 获取脏页面数 */
|
||||
getDirtyPageCount(): number {
|
||||
let count = 0;
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine Buffer Pool — 页面缓存池
|
||||
* @module engine/aria/buffer/pool
|
||||
*
|
||||
* 管理固定数量页面的 LRU 缓存，减少对底层储存的访问。
|
||||
*/
|
||||
|
||||
import type { PageHandle } from '../types';
|
||||
import { PageType, DEFAULT_BUFFER_POOL_PAGES } from '../types';
|
||||
import { createPage } from '../page/format';
|
||||
import { EvictionManager } from './eviction';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Page Read / Write 回调
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface PageIO {
|
||||
/** 从存储后端读取页面 */
|
||||
readPage(pageId: number): Promise<ArrayBuffer | null>;
|
||||
/** 将页面写入存储后端 */
|
||||
writePage(pageId: number, data: ArrayBuffer): Promise<void>;
|
||||
/** 分配新页面 ID */
|
||||
allocatePageId(): Promise<number>;
|
||||
/** v0.4.5: 批量分配页面 ID（一次 meta 持久化） */
|
||||
allocatePageIds?(count: number): Promise<number[]>;
|
||||
/** 释放页面 ID */
|
||||
freePageId(pageId: number): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer Pool
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class BufferPool {
|
||||
private pages: Map<number, PageHandle> = new Map();
|
||||
private eviction: EvictionManager;
|
||||
private pageIO: PageIO;
|
||||
private nextPageId = 0;
|
||||
|
||||
constructor(pageIO: PageIO, capacity: number = DEFAULT_BUFFER_POOL_PAGES) {
|
||||
this.pageIO = pageIO;
|
||||
this.eviction = new EvictionManager(capacity, async (page) => {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 页面获取
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 获取页面（必要时从磁盘读取）。
|
||||
* 返回 pin 的页面，使用完成后必须调用 unpin()。
|
||||
*/
|
||||
async getPage(pageId: number): Promise<PageHandle | null> {
|
||||
// 已在池中
|
||||
let page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.access(page);
|
||||
page.pins++;
|
||||
return page;
|
||||
}
|
||||
|
||||
// 需要从磁盘加载
|
||||
const buffer = await this.pageIO.readPage(pageId);
|
||||
if (!buffer) return null;
|
||||
|
||||
// 确保有空间
|
||||
await this.eviction.evictIfNeeded(1);
|
||||
|
||||
const type = new DataView(buffer).getUint8(4) as PageType;
|
||||
page = {
|
||||
pageId,
|
||||
type,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 1,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.4.5: 批量创建新页面（一次页面 ID 分配，页面化 SSTable 保存用）。
|
||||
* 返回的页面均 pin 且 dirty=false（调用方写入后需 markDirty + flushPage）。
|
||||
*/
|
||||
async newPages(count: number, type: PageType = PageType.DATA): Promise<PageHandle[]> {
|
||||
if (count <= 0) return [];
|
||||
let pageIds: number[];
|
||||
if (typeof this.pageIO.allocatePageIds === 'function') {
|
||||
pageIds = await this.pageIO.allocatePageIds!(count);
|
||||
} else {
|
||||
pageIds = [];
|
||||
for (let i = 0; i < count; i++) pageIds.push(await this.pageIO.allocatePageId());
|
||||
}
|
||||
|
||||
await this.eviction.evictIfNeeded(count);
|
||||
|
||||
const handles: PageHandle[] = [];
|
||||
for (const pageId of pageIds) {
|
||||
const page = createPage(pageId, type);
|
||||
page.pins = 1;
|
||||
this.pages.set(pageId, page);
|
||||
this.eviction.add(page);
|
||||
handles.push(page);
|
||||
}
|
||||
return handles;
|
||||
}
|
||||
|
||||
/**
|
||||
* 释放页面的 pin。
|
||||
*/
|
||||
unpin(page: PageHandle): void {
|
||||
if (page.pins > 0) {
|
||||
page.pins--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记页面为脏（需要写回）。
|
||||
*/
|
||||
markDirty(page: PageHandle): void {
|
||||
page.dirty = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将脏页面刷新到磁盘。
|
||||
*/
|
||||
async flushPage(pageId: number): Promise<void> {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page && page.dirty) {
|
||||
await this.pageIO.writePage(pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新所有脏页面。
|
||||
*/
|
||||
async flushAll(): Promise<void> {
|
||||
for (const [, page] of this.pages) {
|
||||
if (page.dirty) {
|
||||
await this.pageIO.writePage(page.pageId, page.data);
|
||||
page.dirty = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从缓存中删除指定页面（不刷盘）。
|
||||
*/
|
||||
removePage(pageId: number): void {
|
||||
const page = this.pages.get(pageId);
|
||||
if (page) {
|
||||
this.eviction.remove(page);
|
||||
this.pages.delete(pageId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空缓存池（先刷脏页）。
|
||||
*/
|
||||
async clear(): Promise<void> {
|
||||
await this.flushAll();
|
||||
this.pages.clear();
|
||||
this.eviction.clear();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,10 +2,9 @@
|
||||
* AriaEngine LZ4 Compression — 简化 LZ4 压缩/解压
|
||||
* @module engine/aria/compression/lz4
|
||||
*
|
||||
* v0.2.6: 修复往返一致性
|
||||
* - 匹配长度截断到 19 字节（matchField 上限 15 + MIN_MATCH），长匹配分段输出
|
||||
* - 组合 token 的 matchField ∈ [1,15]；matchField=0 且 lo=0 表示末尾纯字面量（无 offset）
|
||||
* - 消除"matchField=0 的组合 token 与纯字面量 token 歧义"
|
||||
* v0.4.5 格式 v2：压缩流前增加 4 字节原始大小头（LE u32），
|
||||
* 解压不再依赖外部估算（高压缩率数据下 buf.length*2 估算不足会截断）。
|
||||
* 旧版压缩数据（无头）视为损坏（compression 选项自 v0.2.6 起已声明不向后兼容）。
|
||||
*
|
||||
* Token 格式（1 字节）:
|
||||
* hi 4bit = litLen (0-15)
|
||||
@@ -17,10 +16,16 @@
|
||||
|
||||
const MIN_MATCH = 4;
|
||||
const MAX_MATCH = MIN_MATCH + 15; // 19，匹配长度上限
|
||||
/** 原始大小头字节数 */
|
||||
const HEADER_SIZE = 4;
|
||||
|
||||
export function compressLZ4(input: Uint8Array): Uint8Array {
|
||||
// 空输入直接返回（无 token 可输出）
|
||||
if (input.byteLength === 0) return input;
|
||||
// 空输入：仅头部（原始大小 0）
|
||||
if (input.byteLength === 0) {
|
||||
const empty = new Uint8Array(HEADER_SIZE);
|
||||
new DataView(empty.buffer).setUint32(0, 0, true);
|
||||
return empty;
|
||||
}
|
||||
|
||||
// 最坏情况：纯字面量分块输出 len/15 个 token + 末尾 token
|
||||
// 上限：len + ceil(len/15) + 8（组合 token 的 offset 开销已包含在内）
|
||||
@@ -73,24 +78,38 @@ export function compressLZ4(input: Uint8Array): Uint8Array {
|
||||
litStart += chunk;
|
||||
}
|
||||
|
||||
// 始终输出压缩流（即使比原数据略大）。
|
||||
// 注意：不能返回原样 input —— 解压端无法区分"压缩流"与"原始数据"，
|
||||
// 原样返回会导致解压器将原始字节误解析为 token（v0.2.6 修复）
|
||||
return out.slice(0, di);
|
||||
// v0.4.5: 前置原始大小头，解压端自描述
|
||||
const stream = out.slice(0, di);
|
||||
const combined = new Uint8Array(HEADER_SIZE + stream.byteLength);
|
||||
new DataView(combined.buffer).setUint32(0, input.byteLength, true);
|
||||
combined.set(stream, HEADER_SIZE);
|
||||
return combined;
|
||||
}
|
||||
|
||||
export function decompressLZ4(input: Uint8Array, originalSize: number): Uint8Array {
|
||||
export function decompressLZ4(input: Uint8Array, _originalSize?: number): Uint8Array {
|
||||
if (input.byteLength < HEADER_SIZE) {
|
||||
throw new Error('LZ4 stream too short: missing header');
|
||||
}
|
||||
const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
|
||||
const originalSize = view.getUint32(0, true);
|
||||
if (originalSize === 0 && input.byteLength === HEADER_SIZE) {
|
||||
return new Uint8Array(0); // 空输入
|
||||
}
|
||||
if (originalSize <= 0 || originalSize > 0x3fffffff) {
|
||||
throw new Error('Invalid LZ4 header: bad original size');
|
||||
}
|
||||
const stream = input.subarray(HEADER_SIZE);
|
||||
const out = new Uint8Array(originalSize);
|
||||
let si = 0, di = 0;
|
||||
|
||||
while (si < input.byteLength && di < originalSize) {
|
||||
const token = input[si++];
|
||||
while (si < stream.byteLength && di < originalSize) {
|
||||
const token = stream[si++];
|
||||
const litLen = (token >> 4) & 0x0F;
|
||||
const matchField = token & 0x0F;
|
||||
|
||||
// 复制字面量
|
||||
for (let i = 0; i < litLen && si < input.byteLength && di < originalSize; i++) {
|
||||
out[di++] = input[si++];
|
||||
for (let i = 0; i < litLen && si < stream.byteLength && di < originalSize; i++) {
|
||||
out[di++] = stream[si++];
|
||||
}
|
||||
|
||||
// matchField=0：纯字面量 token（无 offset 无匹配）。
|
||||
@@ -98,8 +117,8 @@ export function decompressLZ4(input: Uint8Array, originalSize: number): Uint8Arr
|
||||
if (matchField === 0) continue;
|
||||
|
||||
// 组合 token：读取 offset + 复制匹配（可能自重叠）
|
||||
if (si + 1 >= input.byteLength) break;
|
||||
const offset = input[si++] | (input[si++] << 8);
|
||||
if (si + 1 >= stream.byteLength) break;
|
||||
const offset = stream[si++] | (stream[si++] << 8);
|
||||
const matchLen = matchField + MIN_MATCH;
|
||||
for (let i = 0; i < matchLen && di < originalSize; i++) {
|
||||
out[di] = out[di - offset];
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* AriaEngine CRC32 — 标准 CRC-32（IEEE 802.3，多项式 0xEDB88320）
|
||||
* @module engine/aria/crc32
|
||||
*
|
||||
* 查表法实现。
|
||||
* 分段计算约定：
|
||||
* crc32(head + tail) === crc32Finalize(crc32Update(crc32Update(0xFFFFFFFF, head), tail))
|
||||
* === crc32(tail, crc32(head))
|
||||
* 用于 SSTable 文件校验和与 WAL 记录完整性校验。
|
||||
*/
|
||||
|
||||
/** CRC-32 查找表（0xEDB88320 反射多项式） */
|
||||
const CRC32_TABLE = (() => {
|
||||
const table = new Uint32Array(256);
|
||||
for (let i = 0; i < 256; i++) {
|
||||
let c = i;
|
||||
for (let k = 0; k < 8; k++) {
|
||||
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
}
|
||||
table[i] = c >>> 0;
|
||||
}
|
||||
return table;
|
||||
})();
|
||||
|
||||
/** CRC-32 初始累加器状态 */
|
||||
export const CRC32_INIT = 0xffffffff;
|
||||
|
||||
/**
|
||||
* 更新内部 CRC 累加状态（供分段计算使用，接收中间状态返回中间状态）。
|
||||
*/
|
||||
export function crc32Update(state: number, data: Uint8Array): number {
|
||||
let crc = state >>> 0;
|
||||
for (let i = 0; i < data.byteLength; i++) {
|
||||
crc = (CRC32_TABLE[(crc ^ data[i]) & 0xff] ^ (crc >>> 8)) >>> 0;
|
||||
}
|
||||
return crc >>> 0;
|
||||
}
|
||||
|
||||
/** 将中间状态转换为最终校验和（终止计算） */
|
||||
export function crc32Finalize(state: number): number {
|
||||
return (state ^ CRC32_INIT) >>> 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算标准 CRC-32 校验和。
|
||||
* @param data 输入字节
|
||||
* @param seed 前序片段计算的最终校验和（首段传 0 或不传）
|
||||
* @returns 32 位无符号校验和
|
||||
*/
|
||||
export function crc32(data: Uint8Array, seed = 0): number {
|
||||
const state = crc32Update(seed ^ CRC32_INIT, data);
|
||||
return crc32Finalize(state);
|
||||
}
|
||||
|
||||
/** 分段计算便捷函数：crc32Continue(prevFinal, chunk) === crc32(chunk, prevFinal) */
|
||||
export function crc32Continue(prev: number, data: Uint8Array): number {
|
||||
return crc32(data, prev);
|
||||
}
|
||||
@@ -33,18 +33,20 @@ export class CryptoManager {
|
||||
return actualSalt as Uint8Array;
|
||||
}
|
||||
|
||||
async encryptPage(data: ArrayBuffer): Promise<{ iv: Uint8Array; data: ArrayBuffer }> {
|
||||
async encryptPage(data: ArrayBuffer | Uint8Array): Promise<{ iv: Uint8Array; data: ArrayBuffer }> {
|
||||
if (!this.cryptoKey) throw new Error('Crypto not initialized');
|
||||
const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH)) as any;
|
||||
// 传 TypedArray 视图而非裸 ArrayBuffer：SubtleCrypto 通过 ArrayBuffer.isView 检查，
|
||||
// 对跨 realm / 跨 vm 环境的 ArrayBuffer 兼容（Node 18/20 的 webcrypto 对裸 ArrayBuffer 检查严格）
|
||||
const ciphertext = await crypto.subtle.encrypt({ name: ALGO, iv } as any, this.cryptoKey, new Uint8Array(data));
|
||||
const plain = (data instanceof Uint8Array ? data : new Uint8Array(data)) as BufferSource;
|
||||
const ciphertext = await crypto.subtle.encrypt({ name: ALGO, iv } as any, this.cryptoKey, plain);
|
||||
return { iv: iv as Uint8Array, data: ciphertext };
|
||||
}
|
||||
|
||||
async decryptPage(iv: Uint8Array, data: ArrayBuffer): Promise<ArrayBuffer> {
|
||||
async decryptPage(iv: Uint8Array, data: ArrayBuffer | Uint8Array): Promise<ArrayBuffer> {
|
||||
if (!this.cryptoKey) throw new Error('Crypto not initialized');
|
||||
return crypto.subtle.decrypt({ name: ALGO, iv } as any, this.cryptoKey, new Uint8Array(data));
|
||||
const ciphertext = (data instanceof Uint8Array ? data : new Uint8Array(data)) as BufferSource;
|
||||
return crypto.subtle.decrypt({ name: ALGO, iv } as any, this.cryptoKey, ciphertext);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
@@ -53,31 +55,3 @@ export class CryptoManager {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 全局兼容层（旧代码仍可使用全局函数）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const globalCrypto = new CryptoManager();
|
||||
|
||||
/** @deprecated 使用 CryptoManager 实例代替 */
|
||||
export async function initCrypto(password: string, salt?: Uint8Array): Promise<Uint8Array> {
|
||||
return globalCrypto.init(password, salt);
|
||||
}
|
||||
|
||||
/** @deprecated 使用 CryptoManager 实例代替 */
|
||||
export function isCryptoEnabled(): boolean { return globalCrypto.enabled; }
|
||||
|
||||
/** @deprecated 使用 CryptoManager 实例代替 */
|
||||
export async function encryptPage(data: ArrayBuffer): Promise<{ iv: Uint8Array; data: ArrayBuffer }> {
|
||||
return globalCrypto.encryptPage(data);
|
||||
}
|
||||
|
||||
/** @deprecated 使用 CryptoManager 实例代替 */
|
||||
export async function decryptPage(iv: Uint8Array, data: ArrayBuffer): Promise<ArrayBuffer> {
|
||||
return globalCrypto.decryptPage(iv, data);
|
||||
}
|
||||
|
||||
/** @deprecated 使用 CryptoManager 实例代替 */
|
||||
export function closeCrypto(): void {
|
||||
globalCrypto.close();
|
||||
}
|
||||
|
||||
+1848
-1827
File diff suppressed because it is too large
Load Diff
+113
-123
@@ -1,123 +1,113 @@
|
||||
/**
|
||||
* AriaEngine Bloom Filter — 快速判定 key 是否可能存在
|
||||
* @module engine/aria/index/bloom
|
||||
*
|
||||
* 使用双哈希函数 + Kirsch-Mitzenmacher 优化生成 k 个哈希值。
|
||||
*/
|
||||
|
||||
import { DEFAULT_BLOOM_BITS_PER_KEY } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BloomFilter
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class BloomFilter {
|
||||
private bits: Uint8Array;
|
||||
private numHashes: number;
|
||||
private _inserted = 0;
|
||||
|
||||
/**
|
||||
* @param numKeys 预期插入的 key 数量
|
||||
* @param bitsPerKey 每个 key 的位数（默认 10，误报率约 1%）
|
||||
*/
|
||||
constructor(numKeys: number, bitsPerKey: number = DEFAULT_BLOOM_BITS_PER_KEY) {
|
||||
// ceil(numKeys * bitsPerKey / 8)，最少 64 位
|
||||
const numBits = Math.max(64, numKeys * bitsPerKey);
|
||||
const numBytes = Math.ceil(numBits / 8);
|
||||
this.bits = new Uint8Array(numBytes);
|
||||
|
||||
// k = bitsPerKey * ln(2) ≈ bitsPerKey * 0.69
|
||||
this.numHashes = Math.max(1, Math.floor(bitsPerKey * 0.69));
|
||||
}
|
||||
|
||||
/** 从现有数据恢复 */
|
||||
static fromData(data: Uint8Array, numHashes: number): BloomFilter {
|
||||
const bf = new BloomFilter(1); // dummy
|
||||
bf.bits = data;
|
||||
bf.numHashes = numHashes;
|
||||
return bf;
|
||||
}
|
||||
|
||||
/** 插入 key */
|
||||
insert(key: string): void {
|
||||
const hashes = this.getHashes(key);
|
||||
for (const h of hashes) {
|
||||
const byteIdx = Math.floor(h / 8);
|
||||
const bitIdx = h % 8;
|
||||
this.bits[byteIdx] |= (1 << bitIdx);
|
||||
}
|
||||
this._inserted++;
|
||||
}
|
||||
|
||||
/** 检查 key 可能存在（false positive 可能，false negative 不可能） */
|
||||
mayContain(key: string): boolean {
|
||||
const hashes = this.getHashes(key);
|
||||
for (const h of hashes) {
|
||||
const byteIdx = Math.floor(h / 8);
|
||||
const bitIdx = h % 8;
|
||||
if ((this.bits[byteIdx] & (1 << bitIdx)) === 0) {
|
||||
return false; // 确定不存在
|
||||
}
|
||||
}
|
||||
return true; // 可能存在
|
||||
}
|
||||
|
||||
/** 获取序列化数据 */
|
||||
serialize(): Uint8Array {
|
||||
return this.bits;
|
||||
}
|
||||
|
||||
/** bit 数组大小 */
|
||||
getBitSize(): number {
|
||||
return this.bits.byteLength * 8;
|
||||
}
|
||||
|
||||
/** 已插入 key 数量 */
|
||||
getInsertedCount(): number {
|
||||
return this._inserted;
|
||||
}
|
||||
|
||||
/** hash 函数数量 */
|
||||
getHashCount(): number {
|
||||
return this.numHashes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 哈希
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private getHashes(key: string): number[] {
|
||||
const bits = this.bits.byteLength * 8;
|
||||
const h1 = this.fnv1a(key);
|
||||
const h2 = this.murmurSimple(key);
|
||||
|
||||
const hashes: number[] = [];
|
||||
for (let i = 0; i < this.numHashes; i++) {
|
||||
// Kirsch-Mitzenmacher: h_i = h1 + i * h2
|
||||
const h = Math.abs((h1 + i * h2) % bits);
|
||||
hashes.push(h);
|
||||
}
|
||||
return hashes;
|
||||
}
|
||||
|
||||
/** FNV-1a 哈希 */
|
||||
private fnv1a(str: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash ^= str.charCodeAt(i);
|
||||
hash = (hash * 0x01000193) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** 简化的 Murmur-like 哈希 */
|
||||
private murmurSimple(str: string): number {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
const ch = str.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash + ch) | 0;
|
||||
hash = (hash ^ (hash >>> 16)) >>> 0;
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine Bloom Filter — 快速判定 key 是否可能存在
|
||||
* @module engine/aria/index/bloom
|
||||
*
|
||||
* 使用双哈希函数 + Kirsch-Mitzenmacher 优化生成 k 个哈希值。
|
||||
*/
|
||||
|
||||
import { DEFAULT_BLOOM_BITS_PER_KEY } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BloomFilter
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class BloomFilter {
|
||||
private bits: Uint8Array;
|
||||
private numHashes: number;
|
||||
private _inserted = 0;
|
||||
|
||||
/**
|
||||
* @param numKeys 预期插入的 key 数量
|
||||
* @param bitsPerKey 每个 key 的位数（默认 10，误报率约 1%）
|
||||
*/
|
||||
constructor(numKeys: number, bitsPerKey: number = DEFAULT_BLOOM_BITS_PER_KEY) {
|
||||
// ceil(numKeys * bitsPerKey / 8)，最少 64 位
|
||||
const numBits = Math.max(64, numKeys * bitsPerKey);
|
||||
const numBytes = Math.ceil(numBits / 8);
|
||||
this.bits = new Uint8Array(numBytes);
|
||||
|
||||
// k = bitsPerKey * ln(2) ≈ bitsPerKey * 0.69
|
||||
this.numHashes = Math.max(1, Math.floor(bitsPerKey * 0.69));
|
||||
}
|
||||
|
||||
/** 从现有数据恢复 */
|
||||
static fromData(data: Uint8Array, numHashes: number): BloomFilter {
|
||||
const bf = new BloomFilter(1); // dummy
|
||||
bf.bits = data;
|
||||
bf.numHashes = numHashes;
|
||||
return bf;
|
||||
}
|
||||
|
||||
/** 插入 key */
|
||||
insert(key: string): void {
|
||||
const hashes = this.getHashes(key);
|
||||
for (const h of hashes) {
|
||||
const byteIdx = Math.floor(h / 8);
|
||||
const bitIdx = h % 8;
|
||||
this.bits[byteIdx] |= (1 << bitIdx);
|
||||
}
|
||||
this._inserted++;
|
||||
}
|
||||
|
||||
/** 检查 key 可能存在（false positive 可能，false negative 不可能） */
|
||||
mayContain(key: string): boolean {
|
||||
const hashes = this.getHashes(key);
|
||||
for (const h of hashes) {
|
||||
const byteIdx = Math.floor(h / 8);
|
||||
const bitIdx = h % 8;
|
||||
if ((this.bits[byteIdx] & (1 << bitIdx)) === 0) {
|
||||
return false; // 确定不存在
|
||||
}
|
||||
}
|
||||
return true; // 可能存在
|
||||
}
|
||||
|
||||
/** 获取序列化数据 */
|
||||
serialize(): Uint8Array {
|
||||
return this.bits;
|
||||
}
|
||||
|
||||
/** hash 函数数量 */
|
||||
getHashCount(): number {
|
||||
return this.numHashes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 哈希
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private getHashes(key: string): number[] {
|
||||
const bits = this.bits.byteLength * 8;
|
||||
const h1 = this.fnv1a(key);
|
||||
const h2 = this.murmurSimple(key);
|
||||
|
||||
const hashes: number[] = [];
|
||||
for (let i = 0; i < this.numHashes; i++) {
|
||||
// Kirsch-Mitzenmacher: h_i = h1 + i * h2
|
||||
const h = Math.abs((h1 + i * h2) % bits);
|
||||
hashes.push(h);
|
||||
}
|
||||
return hashes;
|
||||
}
|
||||
|
||||
/** FNV-1a 哈希 */
|
||||
private fnv1a(str: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash ^= str.charCodeAt(i);
|
||||
hash = (hash * 0x01000193) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** 简化的 Murmur-like 哈希 */
|
||||
private murmurSimple(str: string): number {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
const ch = str.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash + ch) | 0;
|
||||
hash = (hash ^ (hash >>> 16)) >>> 0;
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
}
|
||||
|
||||
+768
-748
File diff suppressed because it is too large
Load Diff
+462
-467
@@ -1,467 +1,462 @@
|
||||
/**
|
||||
* AriaEngine MemTable — 基于红黑树的内存表
|
||||
* @module engine/aria/index/memtable
|
||||
*
|
||||
* 写操作先进入 MemTable，达到阈值后冻结并 flush 成 SSTable。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RB-Tree Node
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
enum Color { RED, BLACK }
|
||||
|
||||
class RBNode<K, V> {
|
||||
key: K;
|
||||
value: V;
|
||||
color: Color = Color.RED;
|
||||
left: RBNode<K, V> | null = null;
|
||||
right: RBNode<K, V> | null = null;
|
||||
parent: RBNode<K, V> | null = null;
|
||||
|
||||
constructor(key: K, value: V) {
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Red-Black Tree
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class RedBlackTree<K, V> {
|
||||
private root: RBNode<K, V> | null = null;
|
||||
private _size = 0;
|
||||
|
||||
get size(): number { return this._size; }
|
||||
|
||||
// ---- 插入 ----
|
||||
insert(key: K, value: V): void {
|
||||
const node = new RBNode(key, value);
|
||||
|
||||
if (!this.root) {
|
||||
this.root = node;
|
||||
node.color = Color.BLACK;
|
||||
this._size++;
|
||||
return;
|
||||
}
|
||||
|
||||
let parent: RBNode<K, V> | null = null;
|
||||
let current: RBNode<K, V> | null = this.root;
|
||||
|
||||
while (current) {
|
||||
parent = current;
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
// 更新已存在的 key
|
||||
current.value = value;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
node.parent = parent;
|
||||
if (key < parent!.key) {
|
||||
parent!.left = node;
|
||||
} else {
|
||||
parent!.right = node;
|
||||
}
|
||||
|
||||
this._size++;
|
||||
this.fixInsert(node);
|
||||
}
|
||||
|
||||
// ---- 查找 ----
|
||||
find(key: K): V | null {
|
||||
let current = this.root;
|
||||
while (current) {
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
return current.value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- 删除 ----
|
||||
delete(key: K): boolean {
|
||||
// 简化实现：标记删除（实际改为找到并调整树）
|
||||
const node = this.findNode(key);
|
||||
if (!node) return false;
|
||||
|
||||
this.deleteNode(node);
|
||||
this._size--;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- 遍历 ----
|
||||
/** 中序遍历（有序） */
|
||||
inorder(callback: (key: K, value: V) => void): void {
|
||||
this._inorder(this.root, callback);
|
||||
}
|
||||
|
||||
/** 范围遍历 */
|
||||
rangeScan(
|
||||
startKey: K,
|
||||
endKey: K,
|
||||
callback: (key: K, value: V) => void,
|
||||
): void {
|
||||
this._rangeScan(this.root, startKey, endKey, callback);
|
||||
}
|
||||
|
||||
/** 获取所有条目 */
|
||||
getAllEntries(): [K, V][] {
|
||||
const entries: [K, V][] = [];
|
||||
this.inorder((k, v) => entries.push([k, v]));
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.root = null;
|
||||
this._size = 0;
|
||||
}
|
||||
|
||||
// ---- 内部方法 ----
|
||||
|
||||
private findNode(key: K): RBNode<K, V> | null {
|
||||
let current = this.root;
|
||||
while (current) {
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
return current;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private deleteNode(node: RBNode<K, V>): void {
|
||||
// 简化：用左子树最大或右子树最小替换
|
||||
// 完整实现较复杂，这里采用简化策略
|
||||
if (!node.left && !node.right) {
|
||||
this.transplant(node, null);
|
||||
if (node.color === Color.BLACK) this.fixDelete(null, node.parent);
|
||||
} else if (!node.left) {
|
||||
this.transplant(node, node.right);
|
||||
if (node.color === Color.BLACK) this.fixDelete(node.right, node.right!.parent);
|
||||
} else if (!node.right) {
|
||||
this.transplant(node, node.left);
|
||||
if (node.color === Color.BLACK) this.fixDelete(node.left, node.left!.parent);
|
||||
} else {
|
||||
const successor = this.minimum(node.right);
|
||||
if (successor!.parent !== node) {
|
||||
this.transplant(successor!, successor!.right);
|
||||
successor!.right = node.right;
|
||||
successor!.right!.parent = successor;
|
||||
}
|
||||
this.transplant(node, successor);
|
||||
successor!.left = node.left;
|
||||
successor!.left!.parent = successor;
|
||||
const origColor = successor!.color;
|
||||
successor!.color = node.color;
|
||||
if (origColor === Color.BLACK) this.fixDelete(successor!.right, successor!.right?.parent ?? null);
|
||||
}
|
||||
}
|
||||
|
||||
private transplant(u: RBNode<K, V> | null, v: RBNode<K, V> | null): void {
|
||||
if (!u!.parent) {
|
||||
this.root = v;
|
||||
} else if (u === u!.parent.left) {
|
||||
u!.parent.left = v;
|
||||
} else {
|
||||
u!.parent.right = v;
|
||||
}
|
||||
if (v) v.parent = u!.parent;
|
||||
}
|
||||
|
||||
private minimum(node: RBNode<K, V>): RBNode<K, V> {
|
||||
while (node.left) node = node.left;
|
||||
return node;
|
||||
}
|
||||
|
||||
private fixInsert(node: RBNode<K, V>): void {
|
||||
while (node.parent && node.parent.color === Color.RED) {
|
||||
const parent = node.parent;
|
||||
const grandparent = parent.parent;
|
||||
if (!grandparent) break;
|
||||
|
||||
if (parent === grandparent.left) {
|
||||
const uncle = grandparent.right;
|
||||
if (uncle && uncle.color === Color.RED) {
|
||||
parent.color = Color.BLACK;
|
||||
uncle.color = Color.BLACK;
|
||||
grandparent.color = Color.RED;
|
||||
node = grandparent;
|
||||
} else {
|
||||
if (node === parent.right) {
|
||||
node = parent;
|
||||
this.rotateLeft(node);
|
||||
}
|
||||
if (node.parent) node.parent.color = Color.BLACK;
|
||||
if (node.parent?.parent) node.parent.parent.color = Color.RED;
|
||||
if (node.parent?.parent) this.rotateRight(node.parent.parent);
|
||||
}
|
||||
} else {
|
||||
const uncle = grandparent.left;
|
||||
if (uncle && uncle.color === Color.RED) {
|
||||
parent.color = Color.BLACK;
|
||||
uncle.color = Color.BLACK;
|
||||
grandparent.color = Color.RED;
|
||||
node = grandparent;
|
||||
} else {
|
||||
if (node === parent.left) {
|
||||
node = parent;
|
||||
this.rotateRight(node);
|
||||
}
|
||||
if (node.parent) node.parent.color = Color.BLACK;
|
||||
if (node.parent?.parent) node.parent.parent.color = Color.RED;
|
||||
if (node.parent?.parent) this.rotateLeft(node.parent.parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.root) this.root.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private fixDelete(x: RBNode<K, V> | null, parent: RBNode<K, V> | null): void {
|
||||
// 标准 RB-Tree 删除修复（修复"双黑"问题）
|
||||
let node = x;
|
||||
let nodeParent = parent;
|
||||
|
||||
while ((!node || node.color === Color.BLACK) && node !== this.root) {
|
||||
if (!nodeParent) break;
|
||||
|
||||
if (node === nodeParent.left) {
|
||||
let sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
|
||||
// Case 1: 兄弟是红色
|
||||
if (sibling.color === Color.RED) {
|
||||
sibling.color = Color.BLACK;
|
||||
nodeParent.color = Color.RED;
|
||||
this.rotateLeft(nodeParent);
|
||||
sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
}
|
||||
|
||||
// Case 2: 兄弟的两个子节点都是黑色
|
||||
const sibLeft = sibling.left;
|
||||
const sibRight = sibling.right;
|
||||
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
|
||||
(!sibRight || sibRight.color === Color.BLACK)) {
|
||||
sibling.color = Color.RED;
|
||||
node = nodeParent;
|
||||
nodeParent = node.parent;
|
||||
} else {
|
||||
// Case 3: 兄弟右子黑色（左子红色）
|
||||
if (!sibRight || sibRight.color === Color.BLACK) {
|
||||
if (sibLeft) sibLeft.color = Color.BLACK;
|
||||
sibling.color = Color.RED;
|
||||
this.rotateRight(sibling);
|
||||
sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
}
|
||||
// Case 4: 兄弟右子红色
|
||||
sibling.color = nodeParent.color;
|
||||
nodeParent.color = Color.BLACK;
|
||||
if (sibling.right) sibling.right.color = Color.BLACK;
|
||||
this.rotateLeft(nodeParent);
|
||||
node = this.root;
|
||||
}
|
||||
} else {
|
||||
// 镜像：node 是父节点的右子
|
||||
let sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
|
||||
if (sibling.color === Color.RED) {
|
||||
sibling.color = Color.BLACK;
|
||||
nodeParent.color = Color.RED;
|
||||
this.rotateRight(nodeParent);
|
||||
sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
}
|
||||
|
||||
const sibLeft = sibling.left;
|
||||
const sibRight = sibling.right;
|
||||
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
|
||||
(!sibRight || sibRight.color === Color.BLACK)) {
|
||||
sibling.color = Color.RED;
|
||||
node = nodeParent;
|
||||
nodeParent = node.parent;
|
||||
} else {
|
||||
if (!sibLeft || sibLeft.color === Color.BLACK) {
|
||||
if (sibRight) sibRight.color = Color.BLACK;
|
||||
sibling.color = Color.RED;
|
||||
this.rotateLeft(sibling);
|
||||
sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
}
|
||||
sibling.color = nodeParent.color;
|
||||
nodeParent.color = Color.BLACK;
|
||||
if (sibling.left) sibling.left.color = Color.BLACK;
|
||||
this.rotateRight(nodeParent);
|
||||
node = this.root;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (node) node.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private rotateLeft(x: RBNode<K, V>): void {
|
||||
const y = x.right;
|
||||
if (!y) return;
|
||||
x.right = y.left;
|
||||
if (y.left) y.left.parent = x;
|
||||
y.parent = x.parent;
|
||||
if (!x.parent) {
|
||||
this.root = y;
|
||||
} else if (x === x.parent.left) {
|
||||
x.parent.left = y;
|
||||
} else {
|
||||
x.parent.right = y;
|
||||
}
|
||||
y.left = x;
|
||||
x.parent = y;
|
||||
}
|
||||
|
||||
private rotateRight(x: RBNode<K, V>): void {
|
||||
const y = x.left;
|
||||
if (!y) return;
|
||||
x.left = y.right;
|
||||
if (y.right) y.right.parent = x;
|
||||
y.parent = x.parent;
|
||||
if (!x.parent) {
|
||||
this.root = y;
|
||||
} else if (x === x.parent.right) {
|
||||
x.parent.right = y;
|
||||
} else {
|
||||
x.parent.left = y;
|
||||
}
|
||||
y.right = x;
|
||||
x.parent = y;
|
||||
}
|
||||
|
||||
private _inorder(node: RBNode<K, V> | null, cb: (k: K, v: V) => void): void {
|
||||
if (!node) return;
|
||||
this._inorder(node.left, cb);
|
||||
cb(node.key, node.value);
|
||||
this._inorder(node.right, cb);
|
||||
}
|
||||
|
||||
private _rangeScan(
|
||||
node: RBNode<K, V> | null,
|
||||
start: K,
|
||||
end: K,
|
||||
cb: (k: K, v: V) => void,
|
||||
): void {
|
||||
if (!node) return;
|
||||
if (node.key > start) this._rangeScan(node.left, start, end, cb);
|
||||
if (node.key >= start && node.key <= end) cb(node.key, node.value);
|
||||
if (node.key < end) this._rangeScan(node.right, start, end, cb);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MemTable
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class MemTable {
|
||||
private tree: RedBlackTree<string, Record<string, unknown>>;
|
||||
private _estimatedSize = 0;
|
||||
private maxSize: number;
|
||||
|
||||
constructor(maxSize: number = 4 * 1024 * 1024) {
|
||||
this.tree = new RedBlackTree();
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
/** 插入或更新 */
|
||||
put(key: string, value: Record<string, unknown>): void {
|
||||
const oldSize = this.estimateEntrySize(key, this.tree.find(key));
|
||||
const newSize = this.estimateEntrySize(key, value);
|
||||
this.tree.insert(key, value);
|
||||
this._estimatedSize += newSize - oldSize;
|
||||
}
|
||||
|
||||
/** 获取 */
|
||||
get(key: string): Record<string, unknown> | null {
|
||||
return this.tree.find(key);
|
||||
}
|
||||
|
||||
/** 删除 */
|
||||
delete(key: string): boolean {
|
||||
const oldVal = this.tree.find(key);
|
||||
if (oldVal) {
|
||||
this._estimatedSize -= this.estimateEntrySize(key, oldVal);
|
||||
}
|
||||
return this.tree.delete(key);
|
||||
}
|
||||
|
||||
/** 是否应刷盘 */
|
||||
shouldFlush(): boolean {
|
||||
return this._estimatedSize >= this.maxSize;
|
||||
}
|
||||
|
||||
/** 获取所有有序条目 */
|
||||
getAllEntries(): [string, Record<string, unknown>][] {
|
||||
return this.tree.getAllEntries();
|
||||
}
|
||||
|
||||
/** 范围扫描 */
|
||||
rangeScan(
|
||||
startKey: string,
|
||||
endKey: string,
|
||||
): [string, Record<string, unknown>][] {
|
||||
const entries: [string, Record<string, unknown>][] = [];
|
||||
this.tree.rangeScan(startKey, endKey, (k, v) => entries.push([k, v]));
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** 条目数 */
|
||||
getEntryCount(): number {
|
||||
return this.tree.size;
|
||||
}
|
||||
|
||||
/** 估计大小（字节） */
|
||||
getEstimatedSize(): number {
|
||||
return this._estimatedSize;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.tree.clear();
|
||||
this._estimatedSize = 0;
|
||||
}
|
||||
|
||||
/** 检查 key 是否存在 */
|
||||
contains(key: string): boolean {
|
||||
return this.tree.find(key) !== null;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private estimateEntrySize(key: string, value: Record<string, unknown> | null): number {
|
||||
if (!value) return 0;
|
||||
let size = key.length * 2; // UTF-16
|
||||
for (const entry of Object.entries(value)) {
|
||||
size += entry[0].length * 2;
|
||||
const v = entry[1];
|
||||
if (typeof v === 'string') size += v.length * 2;
|
||||
else if (typeof v === 'number') size += 8;
|
||||
else if (typeof v === 'boolean') size += 1;
|
||||
else if (v === null || v === undefined) size += 1;
|
||||
else size += 16; // rough estimate
|
||||
}
|
||||
return size;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine MemTable — 基于红黑树的内存表
|
||||
* @module engine/aria/index/memtable
|
||||
*
|
||||
* 写操作先进入 MemTable，达到阈值后冻结并 flush 成 SSTable。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RB-Tree Node
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
enum Color { RED, BLACK }
|
||||
|
||||
class RBNode<K, V> {
|
||||
key: K;
|
||||
value: V;
|
||||
color: Color = Color.RED;
|
||||
left: RBNode<K, V> | null = null;
|
||||
right: RBNode<K, V> | null = null;
|
||||
parent: RBNode<K, V> | null = null;
|
||||
|
||||
constructor(key: K, value: V) {
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Red-Black Tree
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class RedBlackTree<K, V> {
|
||||
private root: RBNode<K, V> | null = null;
|
||||
private _size = 0;
|
||||
|
||||
get size(): number { return this._size; }
|
||||
|
||||
// ---- 插入 ----
|
||||
insert(key: K, value: V): void {
|
||||
const node = new RBNode(key, value);
|
||||
|
||||
if (!this.root) {
|
||||
this.root = node;
|
||||
node.color = Color.BLACK;
|
||||
this._size++;
|
||||
return;
|
||||
}
|
||||
|
||||
let parent: RBNode<K, V> | null = null;
|
||||
let current: RBNode<K, V> | null = this.root;
|
||||
|
||||
while (current) {
|
||||
parent = current;
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
// 更新已存在的 key
|
||||
current.value = value;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
node.parent = parent;
|
||||
if (key < parent!.key) {
|
||||
parent!.left = node;
|
||||
} else {
|
||||
parent!.right = node;
|
||||
}
|
||||
|
||||
this._size++;
|
||||
this.fixInsert(node);
|
||||
}
|
||||
|
||||
// ---- 查找 ----
|
||||
find(key: K): V | null {
|
||||
let current = this.root;
|
||||
while (current) {
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
return current.value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- 删除 ----
|
||||
delete(key: K): boolean {
|
||||
// 简化实现：标记删除（实际改为找到并调整树）
|
||||
const node = this.findNode(key);
|
||||
if (!node) return false;
|
||||
|
||||
this.deleteNode(node);
|
||||
this._size--;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- 遍历 ----
|
||||
/** 中序遍历（有序） */
|
||||
inorder(callback: (key: K, value: V) => void): void {
|
||||
this._inorder(this.root, callback);
|
||||
}
|
||||
|
||||
/** 范围遍历 */
|
||||
rangeScan(
|
||||
startKey: K,
|
||||
endKey: K,
|
||||
callback: (key: K, value: V) => void,
|
||||
): void {
|
||||
this._rangeScan(this.root, startKey, endKey, callback);
|
||||
}
|
||||
|
||||
/** 获取所有条目 */
|
||||
getAllEntries(): [K, V][] {
|
||||
const entries: [K, V][] = [];
|
||||
this.inorder((k, v) => entries.push([k, v]));
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.root = null;
|
||||
this._size = 0;
|
||||
}
|
||||
|
||||
// ---- 内部方法 ----
|
||||
|
||||
private findNode(key: K): RBNode<K, V> | null {
|
||||
let current = this.root;
|
||||
while (current) {
|
||||
if (key < current.key) {
|
||||
current = current.left;
|
||||
} else if (key > current.key) {
|
||||
current = current.right;
|
||||
} else {
|
||||
return current;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private deleteNode(node: RBNode<K, V>): void {
|
||||
// 简化：用左子树最大或右子树最小替换
|
||||
// 完整实现较复杂，这里采用简化策略
|
||||
if (!node.left && !node.right) {
|
||||
this.transplant(node, null);
|
||||
if (node.color === Color.BLACK) this.fixDelete(null, node.parent);
|
||||
} else if (!node.left) {
|
||||
this.transplant(node, node.right);
|
||||
if (node.color === Color.BLACK) this.fixDelete(node.right, node.right!.parent);
|
||||
} else if (!node.right) {
|
||||
this.transplant(node, node.left);
|
||||
if (node.color === Color.BLACK) this.fixDelete(node.left, node.left!.parent);
|
||||
} else {
|
||||
const successor = this.minimum(node.right);
|
||||
if (successor!.parent !== node) {
|
||||
this.transplant(successor!, successor!.right);
|
||||
successor!.right = node.right;
|
||||
successor!.right!.parent = successor;
|
||||
}
|
||||
this.transplant(node, successor);
|
||||
successor!.left = node.left;
|
||||
successor!.left!.parent = successor;
|
||||
const origColor = successor!.color;
|
||||
successor!.color = node.color;
|
||||
if (origColor === Color.BLACK) this.fixDelete(successor!.right, successor!.right?.parent ?? null);
|
||||
}
|
||||
}
|
||||
|
||||
private transplant(u: RBNode<K, V> | null, v: RBNode<K, V> | null): void {
|
||||
if (!u!.parent) {
|
||||
this.root = v;
|
||||
} else if (u === u!.parent.left) {
|
||||
u!.parent.left = v;
|
||||
} else {
|
||||
u!.parent.right = v;
|
||||
}
|
||||
if (v) v.parent = u!.parent;
|
||||
}
|
||||
|
||||
private minimum(node: RBNode<K, V>): RBNode<K, V> {
|
||||
while (node.left) node = node.left;
|
||||
return node;
|
||||
}
|
||||
|
||||
private fixInsert(node: RBNode<K, V>): void {
|
||||
while (node.parent && node.parent.color === Color.RED) {
|
||||
const parent = node.parent;
|
||||
const grandparent = parent.parent;
|
||||
if (!grandparent) break;
|
||||
|
||||
if (parent === grandparent.left) {
|
||||
const uncle = grandparent.right;
|
||||
if (uncle && uncle.color === Color.RED) {
|
||||
parent.color = Color.BLACK;
|
||||
uncle.color = Color.BLACK;
|
||||
grandparent.color = Color.RED;
|
||||
node = grandparent;
|
||||
} else {
|
||||
if (node === parent.right) {
|
||||
node = parent;
|
||||
this.rotateLeft(node);
|
||||
}
|
||||
if (node.parent) node.parent.color = Color.BLACK;
|
||||
if (node.parent?.parent) node.parent.parent.color = Color.RED;
|
||||
if (node.parent?.parent) this.rotateRight(node.parent.parent);
|
||||
}
|
||||
} else {
|
||||
const uncle = grandparent.left;
|
||||
if (uncle && uncle.color === Color.RED) {
|
||||
parent.color = Color.BLACK;
|
||||
uncle.color = Color.BLACK;
|
||||
grandparent.color = Color.RED;
|
||||
node = grandparent;
|
||||
} else {
|
||||
if (node === parent.left) {
|
||||
node = parent;
|
||||
this.rotateRight(node);
|
||||
}
|
||||
if (node.parent) node.parent.color = Color.BLACK;
|
||||
if (node.parent?.parent) node.parent.parent.color = Color.RED;
|
||||
if (node.parent?.parent) this.rotateLeft(node.parent.parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.root) this.root.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private fixDelete(x: RBNode<K, V> | null, parent: RBNode<K, V> | null): void {
|
||||
// 标准 RB-Tree 删除修复（修复"双黑"问题）
|
||||
let node = x;
|
||||
let nodeParent = parent;
|
||||
|
||||
while ((!node || node.color === Color.BLACK) && node !== this.root) {
|
||||
if (!nodeParent) break;
|
||||
|
||||
if (node === nodeParent.left) {
|
||||
let sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
|
||||
// Case 1: 兄弟是红色
|
||||
if (sibling.color === Color.RED) {
|
||||
sibling.color = Color.BLACK;
|
||||
nodeParent.color = Color.RED;
|
||||
this.rotateLeft(nodeParent);
|
||||
sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
}
|
||||
|
||||
// Case 2: 兄弟的两个子节点都是黑色
|
||||
const sibLeft = sibling.left;
|
||||
const sibRight = sibling.right;
|
||||
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
|
||||
(!sibRight || sibRight.color === Color.BLACK)) {
|
||||
sibling.color = Color.RED;
|
||||
node = nodeParent;
|
||||
nodeParent = node.parent;
|
||||
} else {
|
||||
// Case 3: 兄弟右子黑色（左子红色）
|
||||
if (!sibRight || sibRight.color === Color.BLACK) {
|
||||
if (sibLeft) sibLeft.color = Color.BLACK;
|
||||
sibling.color = Color.RED;
|
||||
this.rotateRight(sibling);
|
||||
sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
}
|
||||
// Case 4: 兄弟右子红色
|
||||
sibling.color = nodeParent.color;
|
||||
nodeParent.color = Color.BLACK;
|
||||
if (sibling.right) sibling.right.color = Color.BLACK;
|
||||
this.rotateLeft(nodeParent);
|
||||
node = this.root;
|
||||
}
|
||||
} else {
|
||||
// 镜像：node 是父节点的右子
|
||||
let sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
|
||||
if (sibling.color === Color.RED) {
|
||||
sibling.color = Color.BLACK;
|
||||
nodeParent.color = Color.RED;
|
||||
this.rotateRight(nodeParent);
|
||||
sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
}
|
||||
|
||||
const sibLeft = sibling.left;
|
||||
const sibRight = sibling.right;
|
||||
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
|
||||
(!sibRight || sibRight.color === Color.BLACK)) {
|
||||
sibling.color = Color.RED;
|
||||
node = nodeParent;
|
||||
nodeParent = node.parent;
|
||||
} else {
|
||||
if (!sibLeft || sibLeft.color === Color.BLACK) {
|
||||
if (sibRight) sibRight.color = Color.BLACK;
|
||||
sibling.color = Color.RED;
|
||||
this.rotateLeft(sibling);
|
||||
sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
}
|
||||
sibling.color = nodeParent.color;
|
||||
nodeParent.color = Color.BLACK;
|
||||
if (sibling.left) sibling.left.color = Color.BLACK;
|
||||
this.rotateRight(nodeParent);
|
||||
node = this.root;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (node) node.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private rotateLeft(x: RBNode<K, V>): void {
|
||||
const y = x.right;
|
||||
if (!y) return;
|
||||
x.right = y.left;
|
||||
if (y.left) y.left.parent = x;
|
||||
y.parent = x.parent;
|
||||
if (!x.parent) {
|
||||
this.root = y;
|
||||
} else if (x === x.parent.left) {
|
||||
x.parent.left = y;
|
||||
} else {
|
||||
x.parent.right = y;
|
||||
}
|
||||
y.left = x;
|
||||
x.parent = y;
|
||||
}
|
||||
|
||||
private rotateRight(x: RBNode<K, V>): void {
|
||||
const y = x.left;
|
||||
if (!y) return;
|
||||
x.left = y.right;
|
||||
if (y.right) y.right.parent = x;
|
||||
y.parent = x.parent;
|
||||
if (!x.parent) {
|
||||
this.root = y;
|
||||
} else if (x === x.parent.right) {
|
||||
x.parent.right = y;
|
||||
} else {
|
||||
x.parent.left = y;
|
||||
}
|
||||
y.right = x;
|
||||
x.parent = y;
|
||||
}
|
||||
|
||||
private _inorder(node: RBNode<K, V> | null, cb: (k: K, v: V) => void): void {
|
||||
if (!node) return;
|
||||
this._inorder(node.left, cb);
|
||||
cb(node.key, node.value);
|
||||
this._inorder(node.right, cb);
|
||||
}
|
||||
|
||||
private _rangeScan(
|
||||
node: RBNode<K, V> | null,
|
||||
start: K,
|
||||
end: K,
|
||||
cb: (k: K, v: V) => void,
|
||||
): void {
|
||||
if (!node) return;
|
||||
if (node.key > start) this._rangeScan(node.left, start, end, cb);
|
||||
if (node.key >= start && node.key <= end) cb(node.key, node.value);
|
||||
if (node.key < end) this._rangeScan(node.right, start, end, cb);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MemTable
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class MemTable {
|
||||
private tree: RedBlackTree<string, Record<string, unknown>>;
|
||||
private _estimatedSize = 0;
|
||||
private maxSize: number;
|
||||
|
||||
constructor(maxSize: number = 4 * 1024 * 1024) {
|
||||
this.tree = new RedBlackTree();
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
/** 插入或更新 */
|
||||
put(key: string, value: Record<string, unknown>): void {
|
||||
const oldSize = this.estimateEntrySize(key, this.tree.find(key));
|
||||
const newSize = this.estimateEntrySize(key, value);
|
||||
this.tree.insert(key, value);
|
||||
this._estimatedSize += newSize - oldSize;
|
||||
}
|
||||
|
||||
/** 获取 */
|
||||
get(key: string): Record<string, unknown> | null {
|
||||
return this.tree.find(key);
|
||||
}
|
||||
|
||||
/** 删除 */
|
||||
delete(key: string): boolean {
|
||||
const oldVal = this.tree.find(key);
|
||||
if (oldVal) {
|
||||
this._estimatedSize -= this.estimateEntrySize(key, oldVal);
|
||||
}
|
||||
return this.tree.delete(key);
|
||||
}
|
||||
|
||||
/** 是否应刷盘 */
|
||||
shouldFlush(): boolean {
|
||||
return this._estimatedSize >= this.maxSize;
|
||||
}
|
||||
|
||||
/** 获取所有有序条目 */
|
||||
getAllEntries(): [string, Record<string, unknown>][] {
|
||||
return this.tree.getAllEntries();
|
||||
}
|
||||
|
||||
/** 范围扫描 */
|
||||
rangeScan(
|
||||
startKey: string,
|
||||
endKey: string,
|
||||
): [string, Record<string, unknown>][] {
|
||||
const entries: [string, Record<string, unknown>][] = [];
|
||||
this.tree.rangeScan(startKey, endKey, (k, v) => entries.push([k, v]));
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** 条目数 */
|
||||
getEntryCount(): number {
|
||||
return this.tree.size;
|
||||
}
|
||||
|
||||
/** 估计大小（字节） */
|
||||
getEstimatedSize(): number {
|
||||
return this._estimatedSize;
|
||||
}
|
||||
|
||||
/** 清空 */
|
||||
clear(): void {
|
||||
this.tree.clear();
|
||||
this._estimatedSize = 0;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private estimateEntrySize(key: string, value: Record<string, unknown> | null): number {
|
||||
if (!value) return 0;
|
||||
let size = key.length * 2; // UTF-16
|
||||
for (const entry of Object.entries(value)) {
|
||||
size += entry[0].length * 2;
|
||||
const v = entry[1];
|
||||
if (typeof v === 'string') size += v.length * 2;
|
||||
else if (typeof v === 'number') size += 8;
|
||||
else if (typeof v === 'boolean') size += 1;
|
||||
else if (v === null || v === undefined) size += 1;
|
||||
else size += 16; // rough estimate
|
||||
}
|
||||
return size;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,192 +1,169 @@
|
||||
/**
|
||||
* AriaEngine Merge Iterator — 多路归并迭代器
|
||||
* @module engine/aria/index/merge_iterator
|
||||
*
|
||||
* 对多个有序 SSTable 或 MemTable 的结果进行归并去重（保留最新值）。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MergeIterator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface EntrySource {
|
||||
/** 获取下一个条目，无更多时返回 null */
|
||||
next(): [string, Record<string, unknown>] | null;
|
||||
/** 重置迭代器 */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
/** 数组数据源的迭代器 */
|
||||
export class ArrayEntrySource implements EntrySource {
|
||||
private entries: [string, Record<string, unknown>][];
|
||||
private index = 0;
|
||||
|
||||
constructor(entries: [string, Record<string, unknown>][]) {
|
||||
this.entries = entries;
|
||||
}
|
||||
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.index >= this.entries.length) return null;
|
||||
return this.entries[this.index++];
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** 回调数据源的迭代器 */
|
||||
export class CallbackEntrySource implements EntrySource {
|
||||
private items: [string, Record<string, unknown>][] = [];
|
||||
private index = 0;
|
||||
private consumed = false;
|
||||
|
||||
/**
|
||||
* @param producer 产生所有条目的回调
|
||||
*/
|
||||
constructor(producer: (cb: (key: string, value: Record<string, unknown>) => void) => void) {
|
||||
producer((key, value) => this.items.push([key, value]));
|
||||
}
|
||||
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.index >= this.items.length) return null;
|
||||
return this.items[this.index++];
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Heap 节点（用于多路归并）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface HeapNode {
|
||||
key: string;
|
||||
value: Record<string, unknown>;
|
||||
sourceIndex: number;
|
||||
}
|
||||
|
||||
/** 最小堆 */
|
||||
class MinHeap {
|
||||
private heap: HeapNode[] = [];
|
||||
|
||||
push(node: HeapNode): void {
|
||||
this.heap.push(node);
|
||||
this.bubbleUp(this.heap.length - 1);
|
||||
}
|
||||
|
||||
pop(): HeapNode | null {
|
||||
if (this.heap.length === 0) return null;
|
||||
if (this.heap.length === 1) return this.heap.pop()!;
|
||||
|
||||
const result = this.heap[0];
|
||||
this.heap[0] = this.heap.pop()!;
|
||||
this.bubbleDown(0);
|
||||
return result;
|
||||
}
|
||||
|
||||
peek(): HeapNode | null {
|
||||
return this.heap.length > 0 ? this.heap[0] : null;
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.heap.length;
|
||||
}
|
||||
|
||||
private bubbleUp(idx: number): void {
|
||||
while (idx > 0) {
|
||||
const parent = Math.floor((idx - 1) / 2);
|
||||
if (this.heap[idx].key >= this.heap[parent].key) break;
|
||||
[this.heap[idx], this.heap[parent]] = [this.heap[parent], this.heap[idx]];
|
||||
idx = parent;
|
||||
}
|
||||
}
|
||||
|
||||
private bubbleDown(idx: number): void {
|
||||
const n = this.heap.length;
|
||||
while (true) {
|
||||
let smallest = idx;
|
||||
const left = 2 * idx + 1;
|
||||
const right = 2 * idx + 2;
|
||||
|
||||
if (left < n && this.heap[left].key < this.heap[smallest].key) smallest = left;
|
||||
if (right < n && this.heap[right].key < this.heap[smallest].key) smallest = right;
|
||||
|
||||
if (smallest === idx) break;
|
||||
|
||||
[this.heap[idx], this.heap[smallest]] = [this.heap[smallest], this.heap[idx]];
|
||||
idx = smallest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MergeIterator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 对多个有序数据源进行归并，重复 key 保留最新（后出现的）。
|
||||
* 数据源按新鲜度排序：越新的数据源在下标越小（如 MemTable 在 SSTable 之前）。
|
||||
*/
|
||||
export class MergeIterator {
|
||||
private sources: EntrySource[];
|
||||
private heap: MinHeap;
|
||||
|
||||
constructor() {
|
||||
this.sources = [];
|
||||
this.heap = new MinHeap();
|
||||
}
|
||||
|
||||
/** 添加数据源 */
|
||||
addSource(source: EntrySource): void {
|
||||
this.sources.push(source);
|
||||
this.seedFromSource(this.sources.length - 1);
|
||||
}
|
||||
|
||||
/** 获取下一个归并后的条目 */
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.heap.size === 0) return null;
|
||||
|
||||
const first = this.heap.pop()!;
|
||||
const key = first.key;
|
||||
let best = first;
|
||||
|
||||
// 刷新 first 来源的下一个值
|
||||
this.seedFromSource(first.sourceIndex);
|
||||
|
||||
// 跳过重复 key：在多个来源中保留 sourceIndex 最小（最新）的条目
|
||||
while (this.heap.peek() && this.heap.peek()!.key === key) {
|
||||
const dup = this.heap.pop()!;
|
||||
this.seedFromSource(dup.sourceIndex);
|
||||
if (dup.sourceIndex < best.sourceIndex) {
|
||||
best = dup;
|
||||
}
|
||||
}
|
||||
|
||||
return [best.key, best.value];
|
||||
}
|
||||
|
||||
/** 耗尽管道，返回所有归并结果 */
|
||||
drain(): [string, Record<string, unknown>][] {
|
||||
const result: [string, Record<string, unknown>][] = [];
|
||||
let entry = this.next();
|
||||
while (entry) {
|
||||
result.push(entry);
|
||||
entry = this.next();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private seedFromSource(sourceIndex: number): void {
|
||||
const entry = this.sources[sourceIndex].next();
|
||||
if (entry) {
|
||||
this.heap.push({
|
||||
key: entry[0],
|
||||
value: entry[1],
|
||||
sourceIndex,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine Merge Iterator — 多路归并迭代器
|
||||
* @module engine/aria/index/merge_iterator
|
||||
*
|
||||
* 对多个有序 SSTable 或 MemTable 的结果进行归并去重（保留最新值）。
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MergeIterator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface EntrySource {
|
||||
/** 获取下一个条目，无更多时返回 null */
|
||||
next(): [string, Record<string, unknown>] | null;
|
||||
/** 重置迭代器 */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
/** 数组数据源的迭代器 */
|
||||
export class ArrayEntrySource implements EntrySource {
|
||||
private entries: [string, Record<string, unknown>][];
|
||||
private index = 0;
|
||||
|
||||
constructor(entries: [string, Record<string, unknown>][]) {
|
||||
this.entries = entries;
|
||||
}
|
||||
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.index >= this.entries.length) return null;
|
||||
return this.entries[this.index++];
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Heap 节点（用于多路归并）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface HeapNode {
|
||||
key: string;
|
||||
value: Record<string, unknown>;
|
||||
sourceIndex: number;
|
||||
}
|
||||
|
||||
/** 最小堆 */
|
||||
class MinHeap {
|
||||
private heap: HeapNode[] = [];
|
||||
|
||||
push(node: HeapNode): void {
|
||||
this.heap.push(node);
|
||||
this.bubbleUp(this.heap.length - 1);
|
||||
}
|
||||
|
||||
pop(): HeapNode | null {
|
||||
if (this.heap.length === 0) return null;
|
||||
if (this.heap.length === 1) return this.heap.pop()!;
|
||||
|
||||
const result = this.heap[0];
|
||||
this.heap[0] = this.heap.pop()!;
|
||||
this.bubbleDown(0);
|
||||
return result;
|
||||
}
|
||||
|
||||
peek(): HeapNode | null {
|
||||
return this.heap.length > 0 ? this.heap[0] : null;
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.heap.length;
|
||||
}
|
||||
|
||||
private bubbleUp(idx: number): void {
|
||||
while (idx > 0) {
|
||||
const parent = Math.floor((idx - 1) / 2);
|
||||
if (this.heap[idx].key >= this.heap[parent].key) break;
|
||||
[this.heap[idx], this.heap[parent]] = [this.heap[parent], this.heap[idx]];
|
||||
idx = parent;
|
||||
}
|
||||
}
|
||||
|
||||
private bubbleDown(idx: number): void {
|
||||
const n = this.heap.length;
|
||||
while (true) {
|
||||
let smallest = idx;
|
||||
const left = 2 * idx + 1;
|
||||
const right = 2 * idx + 2;
|
||||
|
||||
if (left < n && this.heap[left].key < this.heap[smallest].key) smallest = left;
|
||||
if (right < n && this.heap[right].key < this.heap[smallest].key) smallest = right;
|
||||
|
||||
if (smallest === idx) break;
|
||||
|
||||
[this.heap[idx], this.heap[smallest]] = [this.heap[smallest], this.heap[idx]];
|
||||
idx = smallest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MergeIterator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 对多个有序数据源进行归并，重复 key 保留最新（后出现的）。
|
||||
* 数据源按新鲜度排序：越新的数据源在下标越小（如 MemTable 在 SSTable 之前）。
|
||||
*/
|
||||
export class MergeIterator {
|
||||
private sources: EntrySource[];
|
||||
private heap: MinHeap;
|
||||
|
||||
constructor() {
|
||||
this.sources = [];
|
||||
this.heap = new MinHeap();
|
||||
}
|
||||
|
||||
/** 添加数据源 */
|
||||
addSource(source: EntrySource): void {
|
||||
this.sources.push(source);
|
||||
this.seedFromSource(this.sources.length - 1);
|
||||
}
|
||||
|
||||
/** 获取下一个归并后的条目 */
|
||||
next(): [string, Record<string, unknown>] | null {
|
||||
if (this.heap.size === 0) return null;
|
||||
|
||||
const first = this.heap.pop()!;
|
||||
const key = first.key;
|
||||
let best = first;
|
||||
|
||||
// 刷新 first 来源的下一个值
|
||||
this.seedFromSource(first.sourceIndex);
|
||||
|
||||
// 跳过重复 key：在多个来源中保留 sourceIndex 最小（最新）的条目
|
||||
while (this.heap.peek() && this.heap.peek()!.key === key) {
|
||||
const dup = this.heap.pop()!;
|
||||
this.seedFromSource(dup.sourceIndex);
|
||||
if (dup.sourceIndex < best.sourceIndex) {
|
||||
best = dup;
|
||||
}
|
||||
}
|
||||
|
||||
return [best.key, best.value];
|
||||
}
|
||||
|
||||
/** 耗尽管道，返回所有归并结果 */
|
||||
drain(): [string, Record<string, unknown>][] {
|
||||
const result: [string, Record<string, unknown>][] = [];
|
||||
let entry = this.next();
|
||||
while (entry) {
|
||||
result.push(entry);
|
||||
entry = this.next();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private seedFromSource(sourceIndex: number): void {
|
||||
const entry = this.sources[sourceIndex].next();
|
||||
if (entry) {
|
||||
this.heap.push({
|
||||
key: entry[0],
|
||||
value: entry[1],
|
||||
sourceIndex,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
import type { IndexEntry, SSTableMeta } from '../types';
|
||||
import { BloomFilter } from './bloom';
|
||||
import { crc32 } from '../crc32';
|
||||
import { SSTABLE_MAGIC_V1, SSTABLE_MAGIC_V2 } from './sstable_builder';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -23,6 +24,8 @@ export class SSTableReader {
|
||||
private bloomFilter: BloomFilter | null = null;
|
||||
/** 格式版本：1 = u16 长度字段（旧），2 = u32 长度字段（v0.4.4） */
|
||||
private format: 1 | 2 = 2;
|
||||
/** footer 中存储的 checksum（0 = 旧版无校验文件） */
|
||||
private storedChecksum = 0;
|
||||
|
||||
constructor(data: Uint8Array, meta: SSTableMeta) {
|
||||
this.data = data;
|
||||
@@ -31,6 +34,17 @@ export class SSTableReader {
|
||||
this.parseFooter();
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验整文件 CRC-32（覆盖除 checksum 字段外的全部字节）。
|
||||
* checksum === 0 表示旧版文件（v1 / v0.4.4 及更早的 v2），跳过校验返回 true（兼容）。
|
||||
*/
|
||||
verifyChecksum(): boolean {
|
||||
if (this.storedChecksum === 0) return true;
|
||||
if (this.data.byteLength < 4) return false;
|
||||
const computed = crc32(this.data.subarray(0, this.data.byteLength - 4));
|
||||
return computed === this.storedChecksum;
|
||||
}
|
||||
|
||||
/** 长度字段宽度：v2 = 4 字节 u32，v1 = 2 字节 u16 */
|
||||
private lenFieldSize(): number {
|
||||
return this.format === 2 ? 4 : 2;
|
||||
@@ -170,16 +184,6 @@ export class SSTableReader {
|
||||
}
|
||||
}
|
||||
|
||||
/** 获取元数据 */
|
||||
getMeta(): SSTableMeta {
|
||||
return this.meta;
|
||||
}
|
||||
|
||||
/** 获取索引条目数 */
|
||||
getIndexBlockCount(): number {
|
||||
return this.indexEntries.length;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -207,6 +211,7 @@ export class SSTableReader {
|
||||
const bloomSize = this.view.getUint32(footerOffset + 12, false);
|
||||
const bloomHashCount = this.view.getUint32(footerOffset + 16, false);
|
||||
this.entryCount = this.view.getUint32(footerOffset + 20, false);
|
||||
this.storedChecksum = this.view.getUint32(footerOffset + 28, false);
|
||||
|
||||
// v0.4.1-fix: 完整性校验 — 索引块必须完全落在文件内，否则视为残缺文件跳过
|
||||
if (indexOffset + 4 > this.data.byteLength || indexOffset + indexSize > this.data.byteLength) {
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
*/
|
||||
|
||||
import { BloomFilter } from './bloom';
|
||||
import { crc32 } from '../crc32';
|
||||
import type { IndexEntry } from '../types';
|
||||
|
||||
/** v1 格式魔数（"SSTB"，u16 长度字段，兼容旧文件读取） */
|
||||
@@ -137,7 +138,15 @@ export class SSTableBuilder {
|
||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false);
|
||||
view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
|
||||
// checksum 字段先写 0，全部字节就绪后计算整文件 CRC32 再回填
|
||||
view.setUint32(footerOffset + 28, 0, false);
|
||||
|
||||
// v0.4.5: 真实 CRC-32 校验和 — 覆盖除自身（最后 4 字节）外的全部内容。
|
||||
// checksum 永远非 0（计算结果为 0 时用 1 代替），读取端以 0 识别旧版无校验文件
|
||||
const all = new Uint8Array(buf);
|
||||
let checksum = crc32(all.subarray(0, all.byteLength - 4));
|
||||
if (checksum === 0) checksum = 1;
|
||||
view.setUint32(footerOffset + 28, checksum, false);
|
||||
|
||||
return {
|
||||
sstableData: new Uint8Array(buf),
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* AriaEngine Database Lock — 多标签页独占锁（Web Locks API）
|
||||
* @module engine/aria/locks
|
||||
*
|
||||
* v0.4.5: OPFS 等无事务后端缺乏多标签页并发协调，多个标签页同时打开同一库
|
||||
* 会导致写竞态与数据损坏。用 Web Locks API（Chrome 69+ / Firefox 96+ / Safari 15.4+）
|
||||
* 获取库级排他锁：
|
||||
* - ifAvailable 模式：锁被其他标签页持有 → 立即抛 ARIA_LOCKED（不排队挂起）
|
||||
* - 持锁期间回调挂起，close() 时释放
|
||||
* - 浏览器不支持 navigator.locks → 返回 false（如实降级：无并发保护，文档注明）
|
||||
*/
|
||||
|
||||
import { DatabaseError } from '../../constants';
|
||||
|
||||
/** Web Locks 锁名（库级排他） */
|
||||
export function lockName(dbName: string): string {
|
||||
return `metona-sqlark:${dbName}`;
|
||||
}
|
||||
|
||||
export class DatabaseLock {
|
||||
private acquired = false;
|
||||
private supported = false;
|
||||
private releaseResolve: (() => void) | null = null;
|
||||
private releasePromise: Promise<void> | null = null;
|
||||
|
||||
/**
|
||||
* 尝试获取独占锁。
|
||||
* @returns true = 已持锁；false = 环境不支持 Web Locks（无并发保护，调用方可警告）
|
||||
* @throws ARIA_LOCKED 锁被其他标签页持有
|
||||
*/
|
||||
async acquire(dbName: string): Promise<boolean> {
|
||||
const nav = (globalThis as { navigator?: unknown }).navigator as {
|
||||
locks?: { request: (name: string, opts: unknown, cb: (lock: { name: string } | null) => Promise<void> | void) => Promise<void> };
|
||||
} | undefined;
|
||||
const lockManager = nav?.locks;
|
||||
if (!lockManager || typeof lockManager.request !== 'function') {
|
||||
this.supported = false;
|
||||
return false;
|
||||
}
|
||||
this.supported = true;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
// 注意：必须直接调用（不能解构 request —— LockManager 方法依赖 this 绑定）
|
||||
const request = lockManager.request.bind(lockManager);
|
||||
request(
|
||||
lockName(dbName),
|
||||
{ ifAvailable: true, mode: 'exclusive' },
|
||||
async (lock) => {
|
||||
if (!lock) {
|
||||
reject(new DatabaseError(
|
||||
`Database "${dbName}" is already open in another tab (locked)`,
|
||||
'ARIA_LOCKED',
|
||||
));
|
||||
return;
|
||||
}
|
||||
this.acquired = true;
|
||||
const releasePromise = new Promise<void>((res) => { this.releaseResolve = res; });
|
||||
this.releasePromise = releasePromise;
|
||||
// 锁已获取：acquire 返回（open 流程继续）
|
||||
resolve();
|
||||
// 回调挂起：保持锁直到 release() 触发
|
||||
await releasePromise;
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 释放锁（等待回调真正结束，保证锁已归还） */
|
||||
async release(): Promise<void> {
|
||||
if (!this.acquired) return;
|
||||
if (this.releaseResolve) {
|
||||
const res = this.releaseResolve;
|
||||
const p = this.releasePromise!;
|
||||
this.releaseResolve = null;
|
||||
this.releasePromise = null;
|
||||
res();
|
||||
try { await p; } catch { /* 释放过程异常不阻塞 */ }
|
||||
}
|
||||
this.acquired = false;
|
||||
}
|
||||
|
||||
/** 是否已持锁 */
|
||||
isAcquired(): boolean {
|
||||
return this.acquired;
|
||||
}
|
||||
|
||||
/** 环境是否支持 Web Locks */
|
||||
isSupported(): boolean {
|
||||
return this.supported;
|
||||
}
|
||||
}
|
||||
+28
-164
@@ -1,164 +1,28 @@
|
||||
/**
|
||||
* AriaEngine Page Format — 页面格式整合层
|
||||
* @module engine/aria/page/format
|
||||
*
|
||||
* 组合 Header / Slot / Tuple 操作，提供统一的页面管理接口。
|
||||
*/
|
||||
|
||||
import { PAGE_SIZE, PageType, type PageHandle } from '../types';
|
||||
import {
|
||||
initPageHeader,
|
||||
getSlotCount,
|
||||
} from './header';
|
||||
import { allocateSlot, freeSlot, readSlotData } from './slot';
|
||||
import { encodeTuple, decodeTuple } from './tuple';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 页面创建
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 创建一个新的空页面 */
|
||||
export function createPage(pageId: number, type: PageType): PageHandle {
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, type);
|
||||
return {
|
||||
pageId,
|
||||
type,
|
||||
data,
|
||||
dirty: true,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
/** 从 ArrayBuffer 恢复页面句柄 */
|
||||
export function pageFromBuffer(
|
||||
pageId: number,
|
||||
buffer: ArrayBuffer,
|
||||
): PageHandle {
|
||||
return {
|
||||
pageId,
|
||||
type: new DataView(buffer).getUint8(4) as PageType,
|
||||
data: buffer,
|
||||
dirty: false,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 行操作（页面级）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 插入一行到页面，返回 slot 索引，空间不足返回 -1。
|
||||
*/
|
||||
export function pageInsertRow(
|
||||
page: PageHandle,
|
||||
row: Record<string, unknown>,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): number {
|
||||
const encoded = encodeTuple(row, columnOrder, columnTypes);
|
||||
const slotIdx = allocateSlot(page.data, encoded);
|
||||
if (slotIdx >= 0) {
|
||||
page.dirty = true;
|
||||
page.lastAccess = Date.now();
|
||||
}
|
||||
return slotIdx;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取指定 slot 的行数据并解码。
|
||||
*/
|
||||
export function pageReadRow(
|
||||
page: PageHandle,
|
||||
slotIndex: number,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Record<string, unknown> | null {
|
||||
const slotData = readSlotData(page.data, slotIndex);
|
||||
if (!slotData) return null;
|
||||
return decodeTuple(slotData, columnOrder, columnTypes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取页面中的所有行。
|
||||
*/
|
||||
export function pageReadAllRows(
|
||||
page: PageHandle,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Record<string, unknown>[] {
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
const slotCount = getSlotCount(page.data);
|
||||
for (let i = 0; i < slotCount; i++) {
|
||||
const row = pageReadRow(page, i, columnOrder, columnTypes);
|
||||
if (row) rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记 slot 为已删除。
|
||||
*/
|
||||
export function pageDeleteRow(page: PageHandle, slotIndex: number): void {
|
||||
freeSlot(page.data, slotIndex);
|
||||
page.dirty = true;
|
||||
page.lastAccess = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* 重写指定 slot 的行数据。
|
||||
*/
|
||||
export function pageUpdateRow(
|
||||
page: PageHandle,
|
||||
slotIndex: number,
|
||||
row: Record<string, unknown>,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): void {
|
||||
// 先标记旧 slot 为删除
|
||||
pageDeleteRow(page, slotIndex);
|
||||
// 分配新 slot，可能会在不同位置
|
||||
pageInsertRow(page, row, columnOrder, columnTypes);
|
||||
// 注意：调用者需要自行维护 slot index → pk 的映射
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 校验和
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 简单 CRC32（使用预先计算的查找表简化） */
|
||||
export function computeChecksum(data: ArrayBuffer): number {
|
||||
const view = new Uint8Array(data);
|
||||
let hash = 0;
|
||||
for (let i = 0; i < view.byteLength; i++) {
|
||||
hash = ((hash << 5) - hash + view[i]) | 0;
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
/** 更新页面的校验和字段 */
|
||||
export function updateChecksum(page: PageHandle): void {
|
||||
// 先清零校验和字段
|
||||
const view = new DataView(page.data);
|
||||
view.setUint32(11, 0, false);
|
||||
// 计算校验和
|
||||
const cksum = computeChecksum(page.data);
|
||||
view.setUint32(11, cksum, false);
|
||||
}
|
||||
|
||||
/** 验证页面校验和 */
|
||||
export function verifyChecksum(page: PageHandle): boolean {
|
||||
const stored = new DataView(page.data).getUint32(11, false);
|
||||
// 临时清零
|
||||
new DataView(page.data).setUint32(11, 0, false);
|
||||
const computed = computeChecksum(page.data);
|
||||
new DataView(page.data).setUint32(11, stored, false);
|
||||
return stored === computed;
|
||||
}
|
||||
/**
|
||||
* AriaEngine Page Format — 页面创建
|
||||
* @module engine/aria/page/format
|
||||
*
|
||||
* v0.4.5: 仅保留生产代码实际使用的页面创建逻辑。
|
||||
* （Slot Directory / Tuple 编解码曾为行级页面存储设计，但从未接入 LSM 主路径，
|
||||
* 属"宣称页面式但未实现"的半成品，已删除 —— SSTable 以 4KB 页面承载，
|
||||
* 页面内容为原始字节切片，由 PageSSTableStore 管理）
|
||||
*/
|
||||
|
||||
import { PAGE_SIZE, PageType, type PageHandle } from '../types';
|
||||
import { initPageHeader } from './header';
|
||||
|
||||
/** 创建一个新的空页面 */
|
||||
export function createPage(pageId: number, type: PageType): PageHandle {
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, type);
|
||||
return {
|
||||
pageId,
|
||||
type,
|
||||
data,
|
||||
dirty: true,
|
||||
pins: 0,
|
||||
prev: null,
|
||||
next: null,
|
||||
lastAccess: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,97 +1,37 @@
|
||||
/**
|
||||
* AriaEngine Page Header — 页面头部编解码
|
||||
* @module engine/aria/page/header
|
||||
*/
|
||||
|
||||
import { PAGE_HEADER_SIZE, PageType, type PageHeader } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 编码
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 将 PageHeader 编码写入 ArrayBuffer 的前 16 字节。
|
||||
* 布局（大端序）:
|
||||
* [0-3] page_id u32
|
||||
* [4] type u8
|
||||
* [5-6] free_start u16
|
||||
* [7-8] free_end u16
|
||||
* [9-10] slot_count u16
|
||||
* [11-14] checksum u32
|
||||
* [15] reserved u8
|
||||
*/
|
||||
export function encodePageHeader(header: PageHeader, buf: ArrayBuffer): void {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, header.pageId, false);
|
||||
view.setUint8(4, header.type);
|
||||
view.setUint16(5, header.freeStart, false);
|
||||
view.setUint16(7, header.freeEnd, false);
|
||||
view.setUint16(9, header.slotCount, false);
|
||||
view.setUint32(11, header.checksum, false);
|
||||
view.setUint8(15, 0); // reserved
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 ArrayBuffer 解码 PageHeader。
|
||||
*/
|
||||
export function decodePageHeader(buf: ArrayBuffer): PageHeader {
|
||||
const view = new DataView(buf);
|
||||
return {
|
||||
pageId: view.getUint32(0, false),
|
||||
type: view.getUint8(4) as PageType,
|
||||
freeStart: view.getUint16(5, false),
|
||||
freeEnd: view.getUint16(7, false),
|
||||
slotCount: view.getUint16(9, false),
|
||||
checksum: view.getUint32(11, false),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化新页面的 Header。
|
||||
*/
|
||||
export function initPageHeader(
|
||||
buf: ArrayBuffer,
|
||||
pageId: number,
|
||||
type: PageType,
|
||||
): void {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, pageId, false);
|
||||
view.setUint8(4, type);
|
||||
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
|
||||
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
|
||||
view.setUint16(9, 0, false); // slotCount = 0
|
||||
view.setUint32(11, 0, false); // checksum = 0
|
||||
view.setUint8(15, 0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 从 Buffer 中提取 Header 字段的辅助函数
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function getPageType(buf: ArrayBuffer): PageType {
|
||||
return new DataView(buf).getUint8(4) as PageType;
|
||||
}
|
||||
|
||||
export function getPageId(buf: ArrayBuffer): number {
|
||||
return new DataView(buf).getUint32(0, false);
|
||||
}
|
||||
|
||||
export function getSlotCount(buf: ArrayBuffer): number {
|
||||
return new DataView(buf).getUint16(9, false);
|
||||
}
|
||||
|
||||
export function getFreeStart(buf: ArrayBuffer): number {
|
||||
return new DataView(buf).getUint16(5, false);
|
||||
}
|
||||
|
||||
export function setFreeStart(buf: ArrayBuffer, val: number): void {
|
||||
new DataView(buf).setUint16(5, val, false);
|
||||
}
|
||||
|
||||
export function setFreeEnd(buf: ArrayBuffer, val: number): void {
|
||||
new DataView(buf).setUint16(7, val, false);
|
||||
}
|
||||
|
||||
export function setSlotCount(buf: ArrayBuffer, val: number): void {
|
||||
new DataView(buf).setUint16(9, val, false);
|
||||
}
|
||||
/**
|
||||
* AriaEngine Page Header — 页面头初始化
|
||||
* @module engine/aria/page/header
|
||||
*
|
||||
* v0.4.5: 仅保留生产代码实际使用的 initPageHeader。
|
||||
* 页面头部布局（大端序）:
|
||||
* [0-3] page_id u32
|
||||
* [4] type u8
|
||||
* [5-6] free_start u16
|
||||
* [7-8] free_end u16
|
||||
* [9-10] slot_count u16
|
||||
* [11-14] checksum u32
|
||||
* [15] reserved u8
|
||||
*
|
||||
* 注：free_start/free_end/slot_count/checksum 字段为历史行级页面格式遗留，
|
||||
* 页面化 SSTable 使用页面原始字节区（跳过头部），字段保留以维持 16 字节头部对齐。
|
||||
*/
|
||||
|
||||
import { PAGE_HEADER_SIZE } from '../types';
|
||||
|
||||
/**
|
||||
* 初始化新页面的 Header。
|
||||
*/
|
||||
export function initPageHeader(
|
||||
buf: ArrayBuffer,
|
||||
pageId: number,
|
||||
type: number,
|
||||
): void {
|
||||
const view = new DataView(buf);
|
||||
view.setUint32(0, pageId, false);
|
||||
view.setUint8(4, type);
|
||||
view.setUint16(5, PAGE_HEADER_SIZE, false); // freeStart = header 之后
|
||||
view.setUint16(7, buf.byteLength, false); // freeEnd = 页面末尾
|
||||
view.setUint16(9, 0, false); // slotCount = 0
|
||||
view.setUint32(11, 0, false); // checksum = 0
|
||||
view.setUint8(15, 0);
|
||||
}
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
/**
|
||||
* AriaEngine Slot Directory — 页面内行槽位管理
|
||||
* @module engine/aria/page/slot
|
||||
*
|
||||
* Slot 目录从页面 Header 之后向下增长，每条记录 4 字节。
|
||||
*/
|
||||
|
||||
import {
|
||||
PAGE_HEADER_SIZE,
|
||||
SLOT_ENTRY_SIZE,
|
||||
PAGE_SIZE,
|
||||
type SlotEntry,
|
||||
} from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Slot 读写
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 读取 slot 号对应的 SlotEntry。
|
||||
*/
|
||||
export function getSlotEntry(buf: ArrayBuffer, slotIndex: number): SlotEntry {
|
||||
const offset = PAGE_HEADER_SIZE + slotIndex * SLOT_ENTRY_SIZE;
|
||||
const view = new DataView(buf);
|
||||
return {
|
||||
offset: view.getUint16(offset, false),
|
||||
length: view.getUint16(offset + 2, false),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入 slot 号对应的 SlotEntry。
|
||||
*/
|
||||
export function setSlotEntry(
|
||||
buf: ArrayBuffer,
|
||||
slotIndex: number,
|
||||
entry: SlotEntry,
|
||||
): void {
|
||||
const offset = PAGE_HEADER_SIZE + slotIndex * SLOT_ENTRY_SIZE;
|
||||
const view = new DataView(buf);
|
||||
view.setUint16(offset, entry.offset, false);
|
||||
view.setUint16(offset + 2, entry.length, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取页面中所有 slot 条目。
|
||||
*/
|
||||
export function getAllSlots(
|
||||
buf: ArrayBuffer,
|
||||
slotCount: number,
|
||||
): SlotEntry[] {
|
||||
const entries: SlotEntry[] = [];
|
||||
for (let i = 0; i < slotCount; i++) {
|
||||
entries.push(getSlotEntry(buf, i));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Slot 空间计算
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 获取 slot 目录占用的总字节数 */
|
||||
export function getSlotDirectorySize(slotCount: number): number {
|
||||
return slotCount * SLOT_ENTRY_SIZE;
|
||||
}
|
||||
|
||||
/** 获取可用空闲空间（字节） */
|
||||
export function getFreeSpace(buf: ArrayBuffer): number {
|
||||
const view = new DataView(buf);
|
||||
const freeStart = view.getUint16(5, false); // slot 区之后
|
||||
const freeEnd = view.getUint16(7, false); // 数据区之前
|
||||
return freeEnd - freeStart;
|
||||
}
|
||||
|
||||
/** 检查是否有足够空间存放长度为 len 的行 */
|
||||
export function hasEnoughSpace(buf: ArrayBuffer, len: number): boolean {
|
||||
const slotCount = new DataView(buf).getUint16(9, false);
|
||||
const neededSlotSize = (slotCount + 1) * SLOT_ENTRY_SIZE;
|
||||
const freeStart = PAGE_HEADER_SIZE + neededSlotSize;
|
||||
const freeEnd = new DataView(buf).getUint16(7, false);
|
||||
return freeEnd - freeStart >= len;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 插入 / 删除 slot
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 在页面中分配一个 slot 并写入行数据。
|
||||
* 返回分配的 slot 索引，失败返回 -1。
|
||||
*/
|
||||
export function allocateSlot(
|
||||
buf: ArrayBuffer,
|
||||
rowData: Uint8Array,
|
||||
): number {
|
||||
const view = new DataView(buf);
|
||||
const slotCount = view.getUint16(9, false);
|
||||
const neededSlotSpace = (slotCount + 1) * SLOT_ENTRY_SIZE;
|
||||
const freeStart = PAGE_HEADER_SIZE + neededSlotSpace;
|
||||
const freeEnd = view.getUint16(7, false);
|
||||
|
||||
if (freeEnd - freeStart < rowData.byteLength) {
|
||||
return -1; // 空间不足
|
||||
}
|
||||
|
||||
// 将数据放入页面底部
|
||||
const dataOffset = freeEnd - rowData.byteLength;
|
||||
const dest = new Uint8Array(buf, dataOffset, rowData.byteLength);
|
||||
dest.set(rowData);
|
||||
|
||||
// 写入 slot 条目
|
||||
setSlotEntry(buf, slotCount, { offset: dataOffset, length: rowData.byteLength });
|
||||
|
||||
// 更新 header
|
||||
view.setUint16(5, freeStart, false); // freeStart
|
||||
view.setUint16(7, dataOffset, false); // freeEnd
|
||||
view.setUint16(9, slotCount + 1, false); // slotCount
|
||||
|
||||
return slotCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从页面中删除指定 slot 的数据（标记为无效，offset 置 0）。
|
||||
* 注：简化实现，不做 slot 压缩。
|
||||
*/
|
||||
export function freeSlot(buf: ArrayBuffer, slotIndex: number): void {
|
||||
setSlotEntry(buf, slotIndex, { offset: 0, length: 0 });
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩页面槽位：移除已删除 slot，整理碎片空间。
|
||||
* 将有效数据紧凑排列，释放空洞。
|
||||
*/
|
||||
export function compactSlots(buf: ArrayBuffer): number {
|
||||
const view = new DataView(buf);
|
||||
const slotCount = view.getUint16(9, false);
|
||||
if (slotCount === 0) return 0;
|
||||
|
||||
// 收集有效 slot（offset>0 的）
|
||||
const validSlots: { index: number; offset: number; length: number; data: Uint8Array }[] = [];
|
||||
for (let i = 0; i < slotCount; i++) {
|
||||
const entry = getSlotEntry(buf, i);
|
||||
if (entry.offset > 0 && entry.length > 0) {
|
||||
const data = new Uint8Array(buf, entry.offset, entry.length);
|
||||
validSlots.push({ index: i, offset: entry.offset, length: entry.length, data: new Uint8Array(data) });
|
||||
}
|
||||
}
|
||||
|
||||
if (validSlots.length === slotCount) return 0; // 无碎片
|
||||
|
||||
// 从页面底部重新紧凑排列
|
||||
let dataEnd = PAGE_SIZE;
|
||||
const newSlots: { offset: number; length: number }[] = [];
|
||||
|
||||
for (let i = validSlots.length - 1; i >= 0; i--) {
|
||||
const s = validSlots[i];
|
||||
dataEnd -= s.length;
|
||||
new Uint8Array(buf).set(s.data, dataEnd);
|
||||
newSlots.unshift({ offset: dataEnd, length: s.length });
|
||||
}
|
||||
|
||||
// 重写 slot directory
|
||||
view.setUint16(9, validSlots.length, false); // slotCount
|
||||
view.setUint16(7, dataEnd, false); // freeEnd
|
||||
for (let i = 0; i < validSlots.length; i++) {
|
||||
setSlotEntry(buf, i, newSlots[i]);
|
||||
}
|
||||
// 清除剩余 slot 条目
|
||||
for (let i = validSlots.length; i < slotCount; i++) {
|
||||
setSlotEntry(buf, i, { offset: 0, length: 0 });
|
||||
}
|
||||
|
||||
return slotCount - validSlots.length; // 回收的 slot 数量
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取指定 slot 的行数据。
|
||||
*/
|
||||
export function readSlotData(buf: ArrayBuffer, slotIndex: number): Uint8Array | null {
|
||||
const entry = getSlotEntry(buf, slotIndex);
|
||||
if (entry.offset === 0 || entry.length === 0) return null;
|
||||
return new Uint8Array(buf, entry.offset, entry.length);
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
/**
|
||||
* AriaEngine Tuple Codec — 行数据的二进制编解码
|
||||
* @module engine/aria/page/tuple
|
||||
*
|
||||
* 将 Record<string, unknown> 编码为紧凑的二进制格式。
|
||||
*
|
||||
* 格式:
|
||||
* [null bitmap: ceil(colCount/8) bytes]
|
||||
* [column 1 data]
|
||||
* [column 2 data]
|
||||
* ...
|
||||
*
|
||||
* 每列:
|
||||
* type tag (u8) + data
|
||||
* - STRING: [len: u16][UTF-8 bytes]
|
||||
* - NUMBER: [f64: 8 bytes]
|
||||
* - BOOLEAN: [u8: 1 byte]
|
||||
* - DATE: [f64: 8 bytes] (epoch ms)
|
||||
* - JSON: [len: u16][UTF-8 bytes]
|
||||
* - NULL: (no data, just the tag)
|
||||
*/
|
||||
|
||||
import { ColumnEncoding } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 编码
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 将行数据编码为二进制字节数组。
|
||||
* @param row 行数据
|
||||
* @param columnOrder 列名顺序列表（决定编码顺序）
|
||||
* @param columnTypes 列名 → FieldType 映射
|
||||
*/
|
||||
export function encodeTuple(
|
||||
row: Record<string, unknown>,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Uint8Array {
|
||||
// 先计算总大小
|
||||
let size = 0;
|
||||
const nullBitmapBytes = Math.ceil(columnOrder.length / 8);
|
||||
size += nullBitmapBytes;
|
||||
|
||||
// 预计算每列编码后的字节
|
||||
const colData: (Uint8Array | null)[] = [];
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
const col = columnOrder[i];
|
||||
const val = row[col];
|
||||
const encoded = encodeColumn(val, columnTypes[col] ?? 'string');
|
||||
colData.push(encoded);
|
||||
if (encoded) {
|
||||
size += 1 + encoded.byteLength; // tag + data
|
||||
} else {
|
||||
size += 1; // just the NULL tag
|
||||
}
|
||||
}
|
||||
|
||||
const buf = new Uint8Array(size);
|
||||
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
// Null bitmap
|
||||
const nullBitmap = new Uint8Array(nullBitmapBytes);
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
if (colData[i] === null) {
|
||||
nullBitmap[Math.floor(i / 8)] |= (1 << (i % 8));
|
||||
}
|
||||
}
|
||||
buf.set(nullBitmap, offset);
|
||||
offset += nullBitmapBytes;
|
||||
|
||||
// Column data
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
const encoded = colData[i];
|
||||
if (encoded === null) {
|
||||
view.setUint8(offset, ColumnEncoding.NULL);
|
||||
offset += 1;
|
||||
} else {
|
||||
const tag = getEncodingTag(columnTypes[columnOrder[i]] ?? 'string');
|
||||
view.setUint8(offset, tag);
|
||||
offset += 1;
|
||||
buf.set(encoded, offset);
|
||||
offset += encoded.byteLength;
|
||||
}
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编码单个列的值。
|
||||
*/
|
||||
function encodeColumn(value: unknown, fieldType: string): Uint8Array | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
|
||||
switch (fieldType) {
|
||||
case 'string': {
|
||||
const str = String(value);
|
||||
const encoder = new TextEncoder();
|
||||
const bytes = encoder.encode(str);
|
||||
const buf = new Uint8Array(2 + bytes.byteLength);
|
||||
new DataView(buf.buffer).setUint16(0, bytes.byteLength, false);
|
||||
buf.set(bytes, 2);
|
||||
return buf;
|
||||
}
|
||||
case 'number': {
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setFloat64(0, Number(value), false);
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
case 'boolean': {
|
||||
return new Uint8Array([value ? 1 : 0]);
|
||||
}
|
||||
case 'date': {
|
||||
const ts = value instanceof Date ? value.getTime() : new Date(String(value)).getTime();
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setFloat64(0, ts, false);
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
case 'json': {
|
||||
const str = JSON.stringify(value);
|
||||
const encoder = new TextEncoder();
|
||||
const bytes = encoder.encode(str);
|
||||
const buf = new Uint8Array(2 + bytes.byteLength);
|
||||
new DataView(buf.buffer).setUint16(0, bytes.byteLength, false);
|
||||
buf.set(bytes, 2);
|
||||
return buf;
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 解码
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 从二进制字节数组解码行数据。
|
||||
* @returns 行数据，如果格式错误返回 null
|
||||
*/
|
||||
export function decodeTuple(
|
||||
bytes: Uint8Array,
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Record<string, unknown> | null {
|
||||
try {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
const nullBitmapBytes = Math.ceil(columnOrder.length / 8);
|
||||
if (offset + nullBitmapBytes > bytes.byteLength) return null;
|
||||
|
||||
offset += nullBitmapBytes;
|
||||
|
||||
const row: Record<string, unknown> = {};
|
||||
|
||||
for (let i = 0; i < columnOrder.length; i++) {
|
||||
if (offset >= bytes.byteLength) break;
|
||||
|
||||
const tag = view.getUint8(offset);
|
||||
offset += 1;
|
||||
|
||||
if (tag === ColumnEncoding.NULL) {
|
||||
row[columnOrder[i]] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
const col = columnOrder[i];
|
||||
const fType = columnTypes[col] ?? 'string';
|
||||
|
||||
const result = decodeColumnValue(bytes, offset, tag, fType);
|
||||
if (result === null) return null;
|
||||
row[col] = result.value;
|
||||
offset = result.nextOffset;
|
||||
}
|
||||
|
||||
return row;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeColumnValue(
|
||||
bytes: Uint8Array,
|
||||
offset: number,
|
||||
tag: number,
|
||||
_fieldType: string,
|
||||
): { value: unknown; nextOffset: number } | null {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
|
||||
switch (tag) {
|
||||
case ColumnEncoding.STRING:
|
||||
case ColumnEncoding.JSON: {
|
||||
if (offset + 2 > bytes.byteLength) return null;
|
||||
const len = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
if (offset + len > bytes.byteLength) return null;
|
||||
const decoder = new TextDecoder();
|
||||
const str = decoder.decode(bytes.slice(offset, offset + len));
|
||||
return {
|
||||
value: tag === ColumnEncoding.JSON ? JSON.parse(str) : str,
|
||||
nextOffset: offset + len,
|
||||
};
|
||||
}
|
||||
case ColumnEncoding.NUMBER: {
|
||||
if (offset + 8 > bytes.byteLength) return null;
|
||||
const val = view.getFloat64(offset, false);
|
||||
return { value: val, nextOffset: offset + 8 };
|
||||
}
|
||||
case ColumnEncoding.BOOLEAN: {
|
||||
if (offset >= bytes.byteLength) return null;
|
||||
return { value: view.getUint8(offset) !== 0, nextOffset: offset + 1 };
|
||||
}
|
||||
case ColumnEncoding.DATE: {
|
||||
if (offset + 8 > bytes.byteLength) return null;
|
||||
const ts = view.getFloat64(offset, false);
|
||||
return { value: new Date(ts).toISOString(), nextOffset: offset + 8 };
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 辅助
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getEncodingTag(fieldType: string): ColumnEncoding {
|
||||
switch (fieldType) {
|
||||
case 'string': return ColumnEncoding.STRING;
|
||||
case 'number': return ColumnEncoding.NUMBER;
|
||||
case 'boolean': return ColumnEncoding.BOOLEAN;
|
||||
case 'date': return ColumnEncoding.DATE;
|
||||
case 'json': return ColumnEncoding.JSON;
|
||||
default: return ColumnEncoding.NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/** 获取列类型到编码标签的映射 */
|
||||
export function getColumnEncodingMap(
|
||||
columnOrder: string[],
|
||||
columnTypes: Record<string, string>,
|
||||
): Map<string, ColumnEncoding> {
|
||||
const map = new Map<string, ColumnEncoding>();
|
||||
for (const col of columnOrder) {
|
||||
map.set(col, getEncodingTag(columnTypes[col] ?? 'string'));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
@@ -23,6 +23,13 @@ export interface IStorageBackend {
|
||||
read(key: string): Promise<ArrayBuffer | null>;
|
||||
/** 写入数据块 */
|
||||
write(key: string, data: ArrayBuffer): Promise<void>;
|
||||
/**
|
||||
* 追加写入（v0.4.5 WAL 分片用，可选）：
|
||||
* - OPFS 后端实现真追加（createWritable keepExistingData + seek，O(chunk)）
|
||||
* - 未实现的后端由调用方回退 read+write（EncryptedBackend 包装时整体重写保正确性）
|
||||
* 语义：在 key 现有内容末尾追加 data；key 不存在时等同 write。
|
||||
*/
|
||||
append?(key: string, data: ArrayBuffer): Promise<void>;
|
||||
/**
|
||||
* 批量原子写入（v0.4.2-fix）：多个 key 在单个底层事务中提交，
|
||||
* 中断时整体回滚，不留半写状态。WAL count 与记录同事务保证一致性。
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* AriaEngine Encrypted Backend — 全库透明 AES-256-GCM 加密后端
|
||||
* @module engine/aria/store/encrypted_backend
|
||||
*
|
||||
* 装饰器模式包装底层 IStorageBackend：
|
||||
* - 写入时加密（每个 value 独立随机 IV：格式 [12B IV][AES-GCM ciphertext]）
|
||||
* - 读取时解密（GCM 认证标签同时保证完整性）
|
||||
* - WAL / SSTable / Schema / 元数据 全部密文存储（除密钥元数据本身）
|
||||
*
|
||||
* 密钥管理：
|
||||
* - PBKDF2-SHA256（100000 迭代）从密码派生 AES-256-GCM 密钥
|
||||
* - `__aria_keymeta` 明文保存 { salt, verifier }：
|
||||
* - salt：PBKDF2 盐（重启后用同一密码重新派生密钥）
|
||||
* - verifier：对固定明文加密的密文（打开时解密验证密码正确性）
|
||||
* - 密码错误 → GCM 认证失败 → 抛 ARIA_DECRYPT_ERROR
|
||||
*
|
||||
* 依赖浏览器/Node 的 WebCrypto（jest.setup.js 已提供 polyfill）。
|
||||
*/
|
||||
|
||||
import { DatabaseError } from '../../../constants';
|
||||
import type { IStorageBackend } from './backend';
|
||||
import { CryptoManager } from '../crypto';
|
||||
|
||||
const IV_LENGTH = 12;
|
||||
const KEYMETA_KEY = '__aria_keymeta';
|
||||
/** verifier 固定明文（无数据泄露风险） */
|
||||
const VERIFIER_PLAIN = 'metona-sqlark-encryption-verifier';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Base64 工具（浏览器 btoa/atob 与 Node Buffer 双环境）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function bytesToBase64(bytes: Uint8Array): string {
|
||||
if (typeof Buffer !== 'undefined') {
|
||||
return Buffer.from(bytes).toString('base64');
|
||||
}
|
||||
let bin = '';
|
||||
for (let i = 0; i < bytes.byteLength; i++) bin += String.fromCharCode(bytes[i]);
|
||||
return btoa(bin);
|
||||
}
|
||||
|
||||
function base64ToBytes(b64: string): Uint8Array {
|
||||
if (typeof Buffer !== 'undefined') {
|
||||
return new Uint8Array(Buffer.from(b64, 'base64'));
|
||||
}
|
||||
const bin = atob(b64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// EncryptedBackend
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class EncryptedBackend implements IStorageBackend {
|
||||
private crypto: CryptoManager = new CryptoManager();
|
||||
private opened = false;
|
||||
|
||||
constructor(
|
||||
private inner: IStorageBackend,
|
||||
private password: string,
|
||||
) {
|
||||
if (!password || password.length === 0) {
|
||||
throw new DatabaseError('Encryption password must not be empty', 'ARIA_ENCRYPT_CONFIG_ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
/** 底层后端（测试/调试用） */
|
||||
getInner(): IStorageBackend {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/** 加密是否已初始化（打开并验证/创建密钥后为 true） */
|
||||
isCryptoReady(): boolean {
|
||||
return this.crypto.enabled;
|
||||
}
|
||||
|
||||
async open(name: string): Promise<void> {
|
||||
if (!this.inner.isOpen()) {
|
||||
await this.inner.open(name);
|
||||
}
|
||||
|
||||
const raw = await this.inner.read(KEYMETA_KEY);
|
||||
if (raw) {
|
||||
// 已有密钥元数据：用持久化 salt 重新派生并验证密码
|
||||
let meta: { salt?: string; verifier?: string };
|
||||
try {
|
||||
meta = JSON.parse(new TextDecoder().decode(raw)) as { salt?: string; verifier?: string };
|
||||
} catch {
|
||||
throw new DatabaseError('Corrupted encryption key metadata', 'ARIA_DECRYPT_ERROR');
|
||||
}
|
||||
if (!meta.salt || !meta.verifier) {
|
||||
throw new DatabaseError('Corrupted encryption key metadata', 'ARIA_DECRYPT_ERROR');
|
||||
}
|
||||
const salt = base64ToBytes(meta.salt);
|
||||
await this.crypto.init(this.password, salt);
|
||||
const verifierBytes = base64ToBytes(meta.verifier);
|
||||
if (verifierBytes.byteLength <= IV_LENGTH) {
|
||||
throw new DatabaseError('Corrupted encryption key metadata', 'ARIA_DECRYPT_ERROR');
|
||||
}
|
||||
const iv = verifierBytes.subarray(0, IV_LENGTH);
|
||||
const ciphertext = verifierBytes.subarray(IV_LENGTH);
|
||||
try {
|
||||
await this.crypto.decryptPage(iv, ciphertext);
|
||||
} catch {
|
||||
// GCM 认证失败：密码错误（或密钥元数据被篡改）
|
||||
throw new DatabaseError(
|
||||
'Decryption failed: wrong password or corrupted key metadata',
|
||||
'ARIA_DECRYPT_ERROR',
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// 无 keymeta：若库中已存在其他数据 → 明文旧库或 keymeta 丢失，拒绝以加密模式打开
|
||||
const keys = await this.inner.listKeys();
|
||||
if (keys.some((k) => k !== KEYMETA_KEY)) {
|
||||
throw new DatabaseError(
|
||||
'Cannot open with encryption: existing database has no key metadata ' +
|
||||
'(database was created without encryption, or key metadata was lost)',
|
||||
'ARIA_ENCRYPT_CONFIG_ERROR',
|
||||
);
|
||||
}
|
||||
// 新库：生成随机 salt + 派生密钥 + 写入 verifier
|
||||
const salt = await this.crypto.init(this.password);
|
||||
const enc = await this.crypto.encryptPage(new TextEncoder().encode(VERIFIER_PLAIN).buffer);
|
||||
const verifier = new Uint8Array(IV_LENGTH + enc.data.byteLength);
|
||||
verifier.set(enc.iv, 0);
|
||||
verifier.set(new Uint8Array(enc.data), IV_LENGTH);
|
||||
const keymeta = JSON.stringify({
|
||||
salt: bytesToBase64(salt),
|
||||
verifier: bytesToBase64(verifier),
|
||||
});
|
||||
await this.inner.write(KEYMETA_KEY, new TextEncoder().encode(keymeta).buffer);
|
||||
}
|
||||
|
||||
this.opened = true;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.inner.close();
|
||||
this.crypto.close();
|
||||
this.opened = false;
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.opened;
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
this.ensureReady();
|
||||
const raw = await this.inner.read(key);
|
||||
if (raw === null) return null;
|
||||
return this.decrypt(raw);
|
||||
}
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
this.ensureReady();
|
||||
await this.inner.write(key, await this.encrypt(data));
|
||||
}
|
||||
|
||||
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
|
||||
this.ensureReady();
|
||||
const encrypted: Record<string, ArrayBuffer> = {};
|
||||
for (const [key, data] of Object.entries(entries)) {
|
||||
encrypted[key] = await this.encrypt(data);
|
||||
}
|
||||
await this.inner.writeMany(encrypted);
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
this.ensureReady();
|
||||
await this.inner.delete(key);
|
||||
}
|
||||
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
this.ensureReady();
|
||||
await this.inner.deleteMany(keys);
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
this.ensureReady();
|
||||
return this.inner.listKeys();
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
this.ensureReady();
|
||||
return this.inner.exists(key);
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
// 清空全部数据但保留密钥元数据 —— clearAll 语义：库本身保留，密码不失效
|
||||
const keys = await this.inner.listKeys();
|
||||
const toDelete = keys.filter((k) => k !== KEYMETA_KEY);
|
||||
if (toDelete.length > 0) {
|
||||
await this.inner.deleteMany(toDelete);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private ensureReady(): void {
|
||||
if (!this.opened) throw new DatabaseError('EncryptedBackend not opened', 'ARIA_DB_NOT_OPEN');
|
||||
if (!this.crypto.enabled) throw new DatabaseError('EncryptedBackend key not initialized', 'ARIA_DECRYPT_ERROR');
|
||||
}
|
||||
|
||||
/** 加密单块数据：[IV(12)][ciphertext] */
|
||||
private async encrypt(data: ArrayBuffer): Promise<ArrayBuffer> {
|
||||
const enc = await this.crypto.encryptPage(data);
|
||||
const out = new ArrayBuffer(IV_LENGTH + enc.data.byteLength);
|
||||
const outBytes = new Uint8Array(out);
|
||||
outBytes.set(enc.iv, 0);
|
||||
outBytes.set(new Uint8Array(enc.data), IV_LENGTH);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 解密单块数据（GCM 认证失败抛错） */
|
||||
private async decrypt(data: ArrayBuffer): Promise<ArrayBuffer> {
|
||||
const bytes = new Uint8Array(data);
|
||||
if (bytes.byteLength <= IV_LENGTH) {
|
||||
throw new DatabaseError('Corrupted encrypted data block (too short)', 'ARIA_DECRYPT_ERROR');
|
||||
}
|
||||
const iv = bytes.subarray(0, IV_LENGTH);
|
||||
const ciphertext = bytes.subarray(IV_LENGTH);
|
||||
try {
|
||||
return await this.crypto.decryptPage(iv, ciphertext);
|
||||
} catch (error) {
|
||||
if (error instanceof DatabaseError) throw error;
|
||||
throw new DatabaseError('Decryption failed (data corrupted or wrong key)', 'ARIA_DECRYPT_ERROR', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,111 +1,104 @@
|
||||
/**
|
||||
* AriaEngine File Manager — 页面文件管理 + PageIO 实现
|
||||
* @module engine/aria/store/file_manager
|
||||
*
|
||||
* 负责管理页面文件的生命周期：分配/释放页面 ID，读写页面。
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from './backend';
|
||||
import type { PageIO } from '../buffer/pool';
|
||||
import { PAGE_SIZE, PageType } from '../types';
|
||||
import { initPageHeader } from '../page/header';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileManager (implements PageIO)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class FileManager implements PageIO {
|
||||
private backend: IStorageBackend;
|
||||
private nextPageId = 0;
|
||||
private metaLoaded = false;
|
||||
private dbName = '';
|
||||
|
||||
constructor(backend: IStorageBackend) {
|
||||
this.backend = backend;
|
||||
}
|
||||
|
||||
/** 初始化：从存储中读取元数据 */
|
||||
async init(dbName: string): Promise<void> {
|
||||
this.dbName = dbName;
|
||||
const meta = await this.backend.read('__aria_meta');
|
||||
if (meta && meta instanceof ArrayBuffer && meta.byteLength >= 4) {
|
||||
const view = new DataView(meta);
|
||||
this.nextPageId = view.getUint32(0, false);
|
||||
} else {
|
||||
this.nextPageId = 1;
|
||||
await this.saveMeta();
|
||||
}
|
||||
this.metaLoaded = true;
|
||||
}
|
||||
|
||||
// ---- PageIO ----
|
||||
|
||||
async readPage(pageId: number): Promise<ArrayBuffer | null> {
|
||||
const key = `pg_${pageId}`;
|
||||
const data = await this.backend.read(key);
|
||||
if (!data) {
|
||||
// 第一次访问：创建新页面
|
||||
return this.createEmptyPage(pageId, PageType.DATA);
|
||||
}
|
||||
|
||||
// 确保大小正确
|
||||
if (data.byteLength < PAGE_SIZE) {
|
||||
const padded = new ArrayBuffer(PAGE_SIZE);
|
||||
new Uint8Array(padded).set(new Uint8Array(data));
|
||||
return padded;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
async writePage(pageId: number, data: ArrayBuffer): Promise<void> {
|
||||
const key = `pg_${pageId}`;
|
||||
await this.backend.write(key, data);
|
||||
}
|
||||
|
||||
async allocatePageId(): Promise<number> {
|
||||
const id = this.nextPageId++;
|
||||
await this.saveMeta();
|
||||
return id;
|
||||
}
|
||||
|
||||
async freePageId(_pageId: number): Promise<void> {
|
||||
// 简化实现：不回收 pageId
|
||||
const key = `pg_${_pageId}`;
|
||||
await this.backend.delete(key);
|
||||
}
|
||||
|
||||
// ---- 表页面分配 ----
|
||||
|
||||
/**
|
||||
* 分配一个新的表元数据页面。
|
||||
*/
|
||||
async allocateTableRootPage(): Promise<number> {
|
||||
const pageId = await this.allocatePageId();
|
||||
const data = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(data, pageId, PageType.META);
|
||||
await this.writePage(pageId, data);
|
||||
return pageId;
|
||||
}
|
||||
|
||||
// ---- 辅助 ----
|
||||
|
||||
private async saveMeta(): Promise<void> {
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setUint32(0, this.nextPageId, false);
|
||||
await this.backend.write('__aria_meta', buf);
|
||||
}
|
||||
|
||||
private createEmptyPage(pageId: number, type: PageType): ArrayBuffer {
|
||||
const buf = new ArrayBuffer(PAGE_SIZE);
|
||||
initPageHeader(buf, pageId, type);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/** 清空所有数据 */
|
||||
async clearAll(): Promise<void> {
|
||||
await this.backend.clear();
|
||||
this.nextPageId = 1;
|
||||
await this.saveMeta();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine File Manager — 页面文件管理 + PageIO 实现
|
||||
* @module engine/aria/store/file_manager
|
||||
*
|
||||
* 负责管理页面文件的生命周期：分配/释放页面 ID，读写页面。
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from './backend';
|
||||
import type { PageIO } from '../buffer/pool';
|
||||
import { PAGE_SIZE, PageType } from '../types';
|
||||
import { initPageHeader } from '../page/header';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileManager (implements PageIO)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class FileManager implements PageIO {
|
||||
private backend: IStorageBackend;
|
||||
private nextPageId = 0;
|
||||
private metaLoaded = false;
|
||||
private dbName = '';
|
||||
|
||||
constructor(backend: IStorageBackend) {
|
||||
this.backend = backend;
|
||||
}
|
||||
|
||||
/** 初始化：从存储中读取元数据 */
|
||||
async init(dbName: string): Promise<void> {
|
||||
this.dbName = dbName;
|
||||
const meta = await this.backend.read('__aria_meta');
|
||||
if (meta && meta instanceof ArrayBuffer && meta.byteLength >= 4) {
|
||||
const view = new DataView(meta);
|
||||
this.nextPageId = view.getUint32(0, false);
|
||||
} else {
|
||||
this.nextPageId = 1;
|
||||
await this.saveMeta();
|
||||
}
|
||||
this.metaLoaded = true;
|
||||
}
|
||||
|
||||
// ---- PageIO ----
|
||||
|
||||
async readPage(pageId: number): Promise<ArrayBuffer | null> {
|
||||
const key = `pg_${pageId}`;
|
||||
const data = await this.backend.read(key);
|
||||
if (!data) {
|
||||
// v0.4.5: 页面总是先分配（allocatePageId 持久化）后写入 —— 读取缺失页面视为损坏
|
||||
// （此前返回空页面会静默掩盖页面丢失，页面化 SSTable 依赖 null 触发自愈清理）
|
||||
return null;
|
||||
}
|
||||
|
||||
// 确保大小正确
|
||||
if (data.byteLength < PAGE_SIZE) {
|
||||
const padded = new ArrayBuffer(PAGE_SIZE);
|
||||
new Uint8Array(padded).set(new Uint8Array(data));
|
||||
return padded;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
async writePage(pageId: number, data: ArrayBuffer): Promise<void> {
|
||||
const key = `pg_${pageId}`;
|
||||
await this.backend.write(key, data);
|
||||
}
|
||||
|
||||
async allocatePageId(): Promise<number> {
|
||||
const id = this.nextPageId++;
|
||||
await this.saveMeta();
|
||||
return id;
|
||||
}
|
||||
|
||||
/** v0.4.5: 批量分配页面 ID（一次 meta 持久化，避免页面化 SSTable 保存时逐页写 meta） */
|
||||
async allocatePageIds(count: number): Promise<number[]> {
|
||||
if (count <= 0) return [];
|
||||
const ids: number[] = [];
|
||||
const start = this.nextPageId;
|
||||
this.nextPageId += count;
|
||||
for (let i = 0; i < count; i++) ids.push(start + i);
|
||||
await this.saveMeta();
|
||||
return ids;
|
||||
}
|
||||
|
||||
async freePageId(_pageId: number): Promise<void> {
|
||||
// 简化实现：不回收 pageId
|
||||
const key = `pg_${_pageId}`;
|
||||
await this.backend.delete(key);
|
||||
}
|
||||
|
||||
// ---- 辅助 ----
|
||||
|
||||
private async saveMeta(): Promise<void> {
|
||||
const buf = new ArrayBuffer(8);
|
||||
new DataView(buf).setUint32(0, this.nextPageId, false);
|
||||
await this.backend.write('__aria_meta', buf);
|
||||
}
|
||||
|
||||
/** 清空所有数据 */
|
||||
async clearAll(): Promise<void> {
|
||||
await this.backend.clear();
|
||||
this.nextPageId = 1;
|
||||
await this.saveMeta();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,23 @@
|
||||
* 零外部依赖，纯浏览器文件系统 API。
|
||||
* 每个 key 对应 OPFS 目录下的一个二进制文件。
|
||||
*
|
||||
* 浏览器要求：Chrome 102+ / Edge 102+
|
||||
* 原子性与一致性保证（v0.4.5 固化）：
|
||||
* - 单文件 write/append：createWritable 为 copy-on-write —— close 前崩溃旧文件保持不变，
|
||||
* close 后原子替换（单文件写入原子）
|
||||
* - 多文件 writeMany/deleteMany：OPFS 无跨文件事务，串行逐个落盘；调用方（WAL 分片）
|
||||
* 已改为单文件语义，多键操作仅用于一次性的 schema/meta 写入
|
||||
* - 所有写操作串行队列化（同源同进程顺序一致）；单次任务失败不中断队列链，
|
||||
* 错误如实返回给该次调用的调用方
|
||||
* - close() 等待写队列排空后再释放目录句柄（杜绝 close 后挂起写丢失/读旧数据）
|
||||
* - open() 自动清理崩溃残留临时文件（Chromium createWritable 的 .crswap 等）
|
||||
*
|
||||
* 浏览器要求：Chrome 102+ / Edge 102+（Safari 15.2+ / Firefox 111+ 支持基础 OPFS）
|
||||
*/
|
||||
import type { IStorageBackend } from './backend';
|
||||
|
||||
/** 崩溃残留临时文件后缀（createWritable 底层实现可能留下） */
|
||||
const STALE_SUFFIXES = ['.crswap', '.tmp'];
|
||||
|
||||
export class OPFSBackend implements IStorageBackend {
|
||||
private root: FileSystemDirectoryHandle | null = null;
|
||||
private dbDir: FileSystemDirectoryHandle | null = null;
|
||||
@@ -19,9 +32,15 @@ export class OPFSBackend implements IStorageBackend {
|
||||
this.dbName = name;
|
||||
this.root = await navigator.storage.getDirectory();
|
||||
this.dbDir = await this.root.getDirectoryHandle(name, { create: true });
|
||||
// v0.4.5: 清理崩溃残留临时文件（不阻塞打开）
|
||||
await this.cleanupStaleFiles();
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
// v0.4.5-fix: 等待所有挂起写完成（否则 close 后挂起写被静默丢弃/读旧数据）
|
||||
try {
|
||||
await this.writeQueue;
|
||||
} catch { /* 写失败已返回给调用方 */ }
|
||||
this.dbDir = null;
|
||||
this.root = null;
|
||||
}
|
||||
@@ -30,6 +49,23 @@ export class OPFSBackend implements IStorageBackend {
|
||||
return this.dbDir !== null;
|
||||
}
|
||||
|
||||
/** 清理崩溃残留的临时文件（open 时自动调用，repair 也可调用） */
|
||||
async cleanupStaleFiles(): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
try {
|
||||
const dir = this.dbDir as any;
|
||||
const stale: string[] = [];
|
||||
for await (const [name] of dir.entries()) {
|
||||
if (STALE_SUFFIXES.some((s) => name.endsWith(s))) {
|
||||
stale.push(name);
|
||||
}
|
||||
}
|
||||
for (const name of stale) {
|
||||
try { await this.dbDir!.removeEntry(name); } catch { /* ignore */ }
|
||||
}
|
||||
} catch { /* 清理失败不阻塞 */ }
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
if (!this.dbDir) return null;
|
||||
try {
|
||||
@@ -41,21 +77,45 @@ export class OPFSBackend implements IStorageBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 单文件原子写：createWritable 为 copy-on-write，close 后原子替换；
|
||||
* 写入期间崩溃 → 旧文件保持（原子性由浏览器 OPFS 实现保证）。
|
||||
*/
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const run = this.writeQueue.then(async () => {
|
||||
const fh = await this.dbDir!.getFileHandle(key, { create: true });
|
||||
const writable = await fh.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
});
|
||||
return this.writeQueue;
|
||||
// v0.4.5-fix: 单次任务失败不中断队列链（错误仍返回给本次调用方）
|
||||
this.writeQueue = run.then(() => undefined, () => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.4.5: 真追加写 — createWritable(keepExistingData) + seek 到文件末尾。
|
||||
* 单文件 COW 原子（close 前崩溃旧文件保持），无需读旧内容即实现 O(chunk) 追加
|
||||
* （WAL 分片高频写入用）。
|
||||
*/
|
||||
async append(key: string, data: ArrayBuffer): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
const run = this.writeQueue.then(async () => {
|
||||
const fh = await this.dbDir!.getFileHandle(key, { create: true });
|
||||
const existing = await fh.getFile();
|
||||
const writable = await fh.createWritable({ keepExistingData: true });
|
||||
await writable.write({ type: 'write', position: existing.size, data });
|
||||
await writable.close();
|
||||
});
|
||||
this.writeQueue = run.then(() => undefined, () => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
/** v0.4.2-fix: 批量写入 — 串行队列内逐个落盘（OPFS 无跨文件事务，顺序保证一致） */
|
||||
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const run = this.writeQueue.then(async () => {
|
||||
for (const [key, data] of Object.entries(entries)) {
|
||||
const fh = await this.dbDir!.getFileHandle(key, { create: true });
|
||||
const writable = await fh.createWritable();
|
||||
@@ -63,26 +123,29 @@ export class OPFSBackend implements IStorageBackend {
|
||||
await writable.close();
|
||||
}
|
||||
});
|
||||
return this.writeQueue;
|
||||
this.writeQueue = run.then(() => undefined, () => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const run = this.writeQueue.then(async () => {
|
||||
try { await this.dbDir!.removeEntry(key); } catch { /* ignore */ }
|
||||
});
|
||||
return this.writeQueue;
|
||||
this.writeQueue = run.then(() => undefined, () => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
/** v0.4.2-fix: 批量删除 — 串行队列内逐个删除 */
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
if (!this.dbDir) return;
|
||||
this.writeQueue = this.writeQueue.then(async () => {
|
||||
const run = this.writeQueue.then(async () => {
|
||||
for (const key of keys) {
|
||||
try { await this.dbDir!.removeEntry(key); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
return this.writeQueue;
|
||||
this.writeQueue = run.then(() => undefined, () => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* AriaEngine Page SSTable Store — SSTable 页面化物理存储
|
||||
* @module engine/aria/store/page_sstable_store
|
||||
*
|
||||
* v0.4.5: 让 BufferPool/FileManager 真正接入 LSM 读写路径。
|
||||
* SSTable 不再整体存为一个 backend value，而是切分为 4KB 页面：
|
||||
* - 页面由 FileManager 分配 pageId，经 BufferPool 缓存（LRU 驱逐，脏页写回）
|
||||
* - save 语义 = 数据已落盘：页面写入后逐个 flushPage（await 底层写）才返回，
|
||||
* 保证 WAL checkpoint（截断）前 SSTable 数据真实持久化
|
||||
* - 页面 ID 列表经 SSTableMeta.pageIds 持久化；旧 meta（无 pageIds）走整 value 读取
|
||||
*
|
||||
* 与 LSM 的 sstableCache（整文件 LRU）双层缓存并存：
|
||||
* - BufferPool 缓存物理页面（跨 SSTable 共享、受 bufferPoolPages 上限约束）
|
||||
* - LSM 缓存解析后的整文件字节（查询热点复用）
|
||||
*/
|
||||
|
||||
import type { FileManager } from './file_manager';
|
||||
import type { BufferPool } from '../buffer/pool';
|
||||
import type { PageHandle } from '../types';
|
||||
import { PAGE_SIZE, PageType } from '../types';
|
||||
|
||||
export class PageSSTableStore {
|
||||
/** SSTable id → 页面 ID 列表（save 时记录，saveMeta 时注入 meta） */
|
||||
private pageIds = new Map<number, number[]>();
|
||||
|
||||
constructor(
|
||||
private fileManager: FileManager,
|
||||
private bufferPool: BufferPool,
|
||||
) {}
|
||||
|
||||
/** 保存数据：切页 → 写入 BufferPool → 逐页落盘 → 记录 pageIds */
|
||||
async save(id: number, data: Uint8Array): Promise<void> {
|
||||
const pageCount = Math.max(1, Math.ceil(data.byteLength / PAGE_SIZE));
|
||||
const handles: PageHandle[] = await this.bufferPool.newPages(pageCount, PageType.DATA);
|
||||
|
||||
const ids: number[] = [];
|
||||
for (let i = 0; i < pageCount; i++) {
|
||||
const page = handles[i];
|
||||
ids.push(page.pageId);
|
||||
const dest = new Uint8Array(page.data);
|
||||
dest.fill(0); // 清空（最后一页可能不满）
|
||||
const slice = data.subarray(i * PAGE_SIZE, Math.min((i + 1) * PAGE_SIZE, data.byteLength));
|
||||
dest.set(slice, 0);
|
||||
page.dirty = true;
|
||||
// save 语义 = 已持久化：立即落盘（WAL checkpoint 截断依赖此保证）
|
||||
await this.bufferPool.flushPage(page.pageId);
|
||||
this.bufferPool.unpin(page);
|
||||
}
|
||||
this.pageIds.set(id, ids);
|
||||
}
|
||||
|
||||
/** 获取指定 SSTable 的页面 ID 列表（saveMeta 注入用） */
|
||||
getPageIds(id: number): number[] | undefined {
|
||||
return this.pageIds.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按页面 ID 列表读取并拼接为完整字节流。
|
||||
* @param totalSize SSTable 真实大小（meta 持久化）——最后一页可能有 0 填充，按真实大小截断
|
||||
* @returns 缺失页面/读取失败返回 null（调用方视为损坏并清理）
|
||||
*/
|
||||
async load(id: number, pageIds: number[], totalSize: number): Promise<Uint8Array | null> {
|
||||
if (pageIds.length === 0) return null;
|
||||
const chunks: Uint8Array[] = [];
|
||||
for (const pageId of pageIds) {
|
||||
const page = await this.bufferPool.getPage(pageId);
|
||||
if (!page) return null;
|
||||
// 立即复制（后续驱逐安全）
|
||||
chunks.push(new Uint8Array(page.data));
|
||||
this.bufferPool.unpin(page);
|
||||
}
|
||||
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
|
||||
const out = new Uint8Array(Math.min(total, totalSize));
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
const take = Math.min(c.byteLength, out.byteLength - off);
|
||||
if (take <= 0) break;
|
||||
out.set(c.subarray(0, take), off);
|
||||
off += take;
|
||||
}
|
||||
this.pageIds.delete(id);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 释放页面（删除物理页面文件 + 移出 BufferPool） */
|
||||
async delete(id: number, pageIds: number[]): Promise<void> {
|
||||
for (const pageId of pageIds) {
|
||||
this.bufferPool.removePage(pageId);
|
||||
try {
|
||||
await this.fileManager.freePageId(pageId);
|
||||
} catch { /* 清理失败不阻塞 */ }
|
||||
}
|
||||
this.pageIds.delete(id);
|
||||
}
|
||||
}
|
||||
+181
-284
@@ -1,284 +1,181 @@
|
||||
/**
|
||||
* AriaEngine MVCC — 多版本并发控制
|
||||
* @module engine/aria/transaction/mvcc
|
||||
*
|
||||
* 实现快照隔离 (Snapshot Isolation)。
|
||||
* 每个事务看到数据库在事务开始时的快照。
|
||||
*/
|
||||
|
||||
import type { RowVersion, TxnEntry } from '../types';
|
||||
import { TransactionState } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MVCCManager
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class MVCCManager {
|
||||
/** 所有行版本的存储：tableName.key → 版本链 */
|
||||
private versionStore: Map<string, RowVersion[]> = new Map();
|
||||
|
||||
/** 活跃事务表：txnId → TxnEntry */
|
||||
private activeTxns: Map<number, TxnEntry> = new Map();
|
||||
|
||||
/** 事务 ID 计数器 */
|
||||
private nextTxnId = 1;
|
||||
|
||||
/** 全局提交序列号（用于可见性判断） */
|
||||
private globalCommitLsn = 0;
|
||||
|
||||
/**
|
||||
* v0.4.2-fix: 每个事务写入的 tableKey 集合 —
|
||||
* commit/rollback 只遍历本事务写过的 key，避免全库版本链扫描（大表事务 O(N) → O(写入数)）
|
||||
*/
|
||||
private txnWriteKeys: Map<number, Set<string>> = new Map();
|
||||
|
||||
// =======================================================================
|
||||
// 事务管理
|
||||
// =======================================================================
|
||||
|
||||
/** 开始一个事务，返回事务 ID */
|
||||
beginTransaction(): number {
|
||||
const txnId = this.nextTxnId++;
|
||||
this.activeTxns.set(txnId, {
|
||||
txnId,
|
||||
state: TransactionState.ACTIVE,
|
||||
snapshotLsn: this.globalCommitLsn,
|
||||
startTime: Date.now(),
|
||||
});
|
||||
this.txnWriteKeys.set(txnId, new Set());
|
||||
return txnId;
|
||||
}
|
||||
|
||||
/** 提交事务 */
|
||||
commitTransaction(txnId: number): void {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) throw new Error(`Transaction ${txnId} not found`);
|
||||
|
||||
txn.state = TransactionState.COMMITTED;
|
||||
this.globalCommitLsn++;
|
||||
|
||||
// v0.4.2-fix: 仅标记本事务写入的版本（此前遍历全库 versionStore）
|
||||
const writeKeys = this.txnWriteKeys.get(txnId);
|
||||
if (writeKeys) {
|
||||
for (const tableKey of writeKeys) {
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions) continue;
|
||||
for (const version of versions) {
|
||||
if (version.txnId === txnId) {
|
||||
version.committed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理已提交事务的记录
|
||||
this.activeTxns.delete(txnId);
|
||||
this.txnWriteKeys.delete(txnId);
|
||||
}
|
||||
|
||||
/** 回滚事务 */
|
||||
rollbackTransaction(txnId: number): void {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) throw new Error(`Transaction ${txnId} not found`);
|
||||
|
||||
txn.state = TransactionState.ABORTED;
|
||||
|
||||
// v0.4.2-fix: 仅移除本事务写入的版本（此前遍历全库 versionStore）
|
||||
const writeKeys = this.txnWriteKeys.get(txnId);
|
||||
if (writeKeys) {
|
||||
for (const tableKey of writeKeys) {
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions) continue;
|
||||
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||
if (filtered.length === 0) {
|
||||
this.versionStore.delete(tableKey);
|
||||
} else {
|
||||
this.versionStore.set(tableKey, filtered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.activeTxns.delete(txnId);
|
||||
this.txnWriteKeys.delete(txnId);
|
||||
}
|
||||
|
||||
/** 检查事务是否活跃 */
|
||||
isActive(txnId: number): boolean {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
return txn !== undefined && txn.state === TransactionState.ACTIVE;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 版本读写
|
||||
// =======================================================================
|
||||
|
||||
/**
|
||||
* 写入一行（创建新版本）。
|
||||
*/
|
||||
writeVersion(
|
||||
tableName: string,
|
||||
key: string,
|
||||
data: Record<string, unknown>,
|
||||
txnId: number,
|
||||
): void {
|
||||
const tableKey = `${tableName}.${key}`;
|
||||
const versions = this.versionStore.get(tableKey) ?? [];
|
||||
|
||||
const newVersion: RowVersion = {
|
||||
txnId,
|
||||
data,
|
||||
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
|
||||
committed: false,
|
||||
};
|
||||
|
||||
versions.push(newVersion);
|
||||
this.versionStore.set(tableKey, versions);
|
||||
// v0.4.2-fix: 记录本事务写过的 key（commit/rollback 精准清理）
|
||||
this.txnWriteKeys.get(txnId)?.add(tableKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取一行（对指定事务可见的最新版本）。
|
||||
*/
|
||||
readVersion(
|
||||
tableName: string,
|
||||
key: string,
|
||||
txnId: number,
|
||||
): Record<string, unknown> | null {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) return null;
|
||||
|
||||
const tableKey = `${tableName}.${key}`;
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions || versions.length === 0) return null;
|
||||
|
||||
// 从最新版本向前遍历
|
||||
for (let i = versions.length - 1; i >= 0; i--) {
|
||||
const version = versions[i];
|
||||
|
||||
// 1. 如果是当前事务写入的（未提交），可见
|
||||
if (version.txnId === txnId) {
|
||||
return version.data;
|
||||
}
|
||||
|
||||
// 2. 如果是已提交的版本，且在快照 LSN 之前提交，可见
|
||||
if (version.committed) {
|
||||
// 简化：所有已提交版本都可见
|
||||
return version.data;
|
||||
}
|
||||
|
||||
// 3. 其他事务的未提交版本，不可见，继续找更早的版本
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除一行（创建墓碑版本）。
|
||||
*/
|
||||
deleteVersion(tableName: string, key: string, txnId: number): void {
|
||||
this.writeVersion(tableName, key, { __mvcc_tombstone: true } as unknown as Record<string, unknown>, txnId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有行的最新已提交版本（用于非事务读取）。
|
||||
*/
|
||||
getLatestCommittedVersions(
|
||||
tableName: string,
|
||||
): Record<string, Record<string, unknown>> {
|
||||
const result: Record<string, Record<string, unknown>> = {};
|
||||
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (!tableKey.startsWith(`${tableName}.`)) continue;
|
||||
const key = tableKey.slice(tableName.length + 1);
|
||||
|
||||
for (let i = versions.length - 1; i >= 0; i--) {
|
||||
const version = versions[i];
|
||||
if (version.committed && !(version.data as unknown as Record<string, unknown>).__mvcc_tombstone) {
|
||||
result[key] = version.data;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.3.3: 丢弃指定事务的所有版本记录，但保留事务登记（Savepoint 回滚用）。
|
||||
* 快照数据由调用方（引擎 txnSnapshot）负责恢复。
|
||||
* v0.4.2-fix: 仅遍历本事务写过的 key（此前全库扫描）。
|
||||
*/
|
||||
discardVersions(txnId: number): void {
|
||||
const writeKeys = this.txnWriteKeys.get(txnId);
|
||||
if (!writeKeys) return;
|
||||
for (const tableKey of writeKeys) {
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions) continue;
|
||||
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||
if (filtered.length === 0) {
|
||||
this.versionStore.delete(tableKey);
|
||||
} else {
|
||||
this.versionStore.set(tableKey, filtered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理过旧版本（GC）。
|
||||
* 保留每个 key 的最新 N 个已提交版本。
|
||||
*/
|
||||
gc(maxVersionsPerKey: number = 100): void {
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (versions.length <= maxVersionsPerKey) continue;
|
||||
|
||||
// 保留最新的 maxVersionsPerKey 个版本
|
||||
const pruned = versions.slice(versions.length - maxVersionsPerKey);
|
||||
this.versionStore.set(tableKey, pruned);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有未提交事务中的 key 列表。
|
||||
*/
|
||||
getActiveWriteKeys(tableName: string, txnId: number): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
const prefix = `${tableName}.`;
|
||||
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (!tableKey.startsWith(prefix)) continue;
|
||||
const latestVersion = versions[versions.length - 1];
|
||||
if (latestVersion.txnId === txnId && !latestVersion.committed) {
|
||||
keys.add(tableKey.slice(prefix.length));
|
||||
}
|
||||
}
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理指定表的所有版本。
|
||||
*/
|
||||
clearTable(tableName: string): void {
|
||||
const prefix = `${tableName}.`;
|
||||
for (const [tableKey] of this.versionStore) {
|
||||
if (tableKey.startsWith(prefix)) {
|
||||
this.versionStore.delete(tableKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取活跃事务数。
|
||||
*/
|
||||
getActiveTxnCount(): number {
|
||||
return this.activeTxns.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取全局 LSN。
|
||||
*/
|
||||
getGlobalLSN(): number {
|
||||
return this.globalCommitLsn;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine MVCC — 多版本并发控制
|
||||
* @module engine/aria/transaction/mvcc
|
||||
*
|
||||
* 实现快照隔离 (Snapshot Isolation)。
|
||||
* 每个事务看到数据库在事务开始时的快照。
|
||||
*/
|
||||
|
||||
import type { RowVersion, TxnEntry } from '../types';
|
||||
import { TransactionState } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MVCCManager
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class MVCCManager {
|
||||
/** 所有行版本的存储：tableName.key → 版本链 */
|
||||
private versionStore: Map<string, RowVersion[]> = new Map();
|
||||
|
||||
/** 活跃事务表：txnId → TxnEntry */
|
||||
private activeTxns: Map<number, TxnEntry> = new Map();
|
||||
|
||||
/** 事务 ID 计数器 */
|
||||
private nextTxnId = 1;
|
||||
|
||||
/** 全局提交序列号（用于可见性判断） */
|
||||
private globalCommitLsn = 0;
|
||||
|
||||
/**
|
||||
* v0.4.2-fix: 每个事务写入的 tableKey 集合 —
|
||||
* commit/rollback 只遍历本事务写过的 key，避免全库版本链扫描（大表事务 O(N) → O(写入数)）
|
||||
*/
|
||||
private txnWriteKeys: Map<number, Set<string>> = new Map();
|
||||
|
||||
// =======================================================================
|
||||
// 事务管理
|
||||
// =======================================================================
|
||||
|
||||
/** 开始一个事务，返回事务 ID */
|
||||
beginTransaction(): number {
|
||||
const txnId = this.nextTxnId++;
|
||||
this.activeTxns.set(txnId, {
|
||||
txnId,
|
||||
state: TransactionState.ACTIVE,
|
||||
snapshotLsn: this.globalCommitLsn,
|
||||
startTime: Date.now(),
|
||||
});
|
||||
this.txnWriteKeys.set(txnId, new Set());
|
||||
return txnId;
|
||||
}
|
||||
|
||||
/** 提交事务 */
|
||||
commitTransaction(txnId: number): void {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) throw new Error(`Transaction ${txnId} not found`);
|
||||
|
||||
txn.state = TransactionState.COMMITTED;
|
||||
this.globalCommitLsn++;
|
||||
|
||||
// v0.4.2-fix: 仅标记本事务写入的版本（此前遍历全库 versionStore）
|
||||
const writeKeys = this.txnWriteKeys.get(txnId);
|
||||
if (writeKeys) {
|
||||
for (const tableKey of writeKeys) {
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions) continue;
|
||||
for (const version of versions) {
|
||||
if (version.txnId === txnId) {
|
||||
version.committed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理已提交事务的记录
|
||||
this.activeTxns.delete(txnId);
|
||||
this.txnWriteKeys.delete(txnId);
|
||||
}
|
||||
|
||||
/** 回滚事务 */
|
||||
rollbackTransaction(txnId: number): void {
|
||||
const txn = this.activeTxns.get(txnId);
|
||||
if (!txn) throw new Error(`Transaction ${txnId} not found`);
|
||||
|
||||
txn.state = TransactionState.ABORTED;
|
||||
|
||||
// v0.4.2-fix: 仅移除本事务写入的版本（此前遍历全库 versionStore）
|
||||
const writeKeys = this.txnWriteKeys.get(txnId);
|
||||
if (writeKeys) {
|
||||
for (const tableKey of writeKeys) {
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions) continue;
|
||||
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||
if (filtered.length === 0) {
|
||||
this.versionStore.delete(tableKey);
|
||||
} else {
|
||||
this.versionStore.set(tableKey, filtered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.activeTxns.delete(txnId);
|
||||
this.txnWriteKeys.delete(txnId);
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 版本读写
|
||||
// =======================================================================
|
||||
|
||||
/**
|
||||
* 写入一行（创建新版本）。
|
||||
*/
|
||||
writeVersion(
|
||||
tableName: string,
|
||||
key: string,
|
||||
data: Record<string, unknown>,
|
||||
txnId: number,
|
||||
): void {
|
||||
const tableKey = `${tableName}.${key}`;
|
||||
const versions = this.versionStore.get(tableKey) ?? [];
|
||||
|
||||
const newVersion: RowVersion = {
|
||||
txnId,
|
||||
data,
|
||||
prevVersion: versions.length > 0 ? versions[versions.length - 1] : null,
|
||||
committed: false,
|
||||
};
|
||||
|
||||
versions.push(newVersion);
|
||||
this.versionStore.set(tableKey, versions);
|
||||
// v0.4.2-fix: 记录本事务写过的 key（commit/rollback 精准清理）
|
||||
this.txnWriteKeys.get(txnId)?.add(tableKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除一行（创建墓碑版本）。
|
||||
*/
|
||||
deleteVersion(tableName: string, key: string, txnId: number): void {
|
||||
this.writeVersion(tableName, key, { __mvcc_tombstone: true } as unknown as Record<string, unknown>, txnId);
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.3.3: 丢弃指定事务的所有版本记录，但保留事务登记（Savepoint 回滚用）。
|
||||
* 快照数据由调用方（引擎 txnSnapshot）负责恢复。
|
||||
* v0.4.2-fix: 仅遍历本事务写过的 key（此前全库扫描）。
|
||||
*/
|
||||
discardVersions(txnId: number): void {
|
||||
const writeKeys = this.txnWriteKeys.get(txnId);
|
||||
if (!writeKeys) return;
|
||||
for (const tableKey of writeKeys) {
|
||||
const versions = this.versionStore.get(tableKey);
|
||||
if (!versions) continue;
|
||||
const filtered = versions.filter((v) => v.txnId !== txnId);
|
||||
if (filtered.length === 0) {
|
||||
this.versionStore.delete(tableKey);
|
||||
} else {
|
||||
this.versionStore.set(tableKey, filtered);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理过旧版本（GC）。
|
||||
* 保留每个 key 的最新 N 个已提交版本。
|
||||
*/
|
||||
gc(maxVersionsPerKey: number = 100): void {
|
||||
for (const [tableKey, versions] of this.versionStore) {
|
||||
if (versions.length <= maxVersionsPerKey) continue;
|
||||
|
||||
// 保留最新的 maxVersionsPerKey 个版本
|
||||
const pruned = versions.slice(versions.length - maxVersionsPerKey);
|
||||
this.versionStore.set(tableKey, pruned);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取全局 LSN。
|
||||
*/
|
||||
getGlobalLSN(): number {
|
||||
return this.globalCommitLsn;
|
||||
}
|
||||
}
|
||||
|
||||
+249
-302
@@ -1,302 +1,249 @@
|
||||
/**
|
||||
* AriaEngine Types — 内部类型定义
|
||||
* @module engine/aria/types
|
||||
*
|
||||
* 页面式存储引擎的所有内部枚举、接口和常量。
|
||||
*/
|
||||
|
||||
// =============================================================================
|
||||
// 页面常量
|
||||
// =============================================================================
|
||||
|
||||
/** 页面大小：4KB */
|
||||
export const PAGE_SIZE = 4096;
|
||||
|
||||
/** 页面头大小：16 字节 */
|
||||
export const PAGE_HEADER_SIZE = 16;
|
||||
|
||||
/** 每个 Slot 目录项大小：4 字节 (offset: u16 + len: u16) */
|
||||
export const SLOT_ENTRY_SIZE = 4;
|
||||
|
||||
/** 页面数据区起始偏移（头部之后） */
|
||||
export const PAGE_DATA_START = PAGE_HEADER_SIZE;
|
||||
|
||||
/** 无效页面 ID */
|
||||
export const INVALID_PAGE_ID = 0xFFFFFFFF;
|
||||
|
||||
// =============================================================================
|
||||
// 页面类型
|
||||
// =============================================================================
|
||||
|
||||
export enum PageType {
|
||||
/** 数据页面：存储行数据 */
|
||||
DATA = 1,
|
||||
/** 索引页面：存储索引节点 */
|
||||
INDEX = 2,
|
||||
/** 溢出页面：存储大字段 */
|
||||
OVERFLOW = 3,
|
||||
/** 元数据页面：存储表/库元信息 */
|
||||
META = 4,
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 页面头部（16 字节）
|
||||
// =============================================================================
|
||||
|
||||
export interface PageHeader {
|
||||
/** 页面 ID（全局唯一） */
|
||||
pageId: number;
|
||||
/** 页面类型 */
|
||||
type: PageType;
|
||||
/** 空闲空间起始偏移（slot 区结束位置） */
|
||||
freeStart: number;
|
||||
/** 数据区结束偏移（从页面底部向上增长） */
|
||||
freeEnd: number;
|
||||
/** 当前 slot 数量 */
|
||||
slotCount: number;
|
||||
/** CRC32 校验和 */
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Slot 目录项（4 字节）
|
||||
// =============================================================================
|
||||
|
||||
export interface SlotEntry {
|
||||
/** 行数据在页面内的偏移 */
|
||||
offset: number;
|
||||
/** 行数据长度（字节） */
|
||||
length: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 页面句柄（Buffer Pool 中的页面）
|
||||
// =============================================================================
|
||||
|
||||
export interface PageHandle {
|
||||
/** 页面 ID */
|
||||
pageId: number;
|
||||
/** 页面类型 */
|
||||
type: PageType;
|
||||
/** 页面数据缓冲区（4KB ArrayBuffer） */
|
||||
data: ArrayBuffer;
|
||||
/** 是否被修改（脏页） */
|
||||
dirty: boolean;
|
||||
/** 引用计数（pin count） */
|
||||
pins: number;
|
||||
/** LRU 链表前驱 */
|
||||
prev: PageHandle | null;
|
||||
/** LRU 链表后继 */
|
||||
next: PageHandle | null;
|
||||
/** 最后访问时间戳 */
|
||||
lastAccess: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 行编解码
|
||||
// =============================================================================
|
||||
|
||||
/** 行/元组的二进制表示 */
|
||||
export interface SerializedTuple {
|
||||
/** 序列化后的字节数组 */
|
||||
bytes: Uint8Array;
|
||||
/** 该行中 null 列的位图 */
|
||||
nullBitmap: Uint8Array;
|
||||
}
|
||||
|
||||
/** 列类型（内部二进制编码用） */
|
||||
export enum ColumnEncoding {
|
||||
STRING = 1,
|
||||
NUMBER = 2,
|
||||
BOOLEAN = 3,
|
||||
DATE = 4,
|
||||
JSON = 5,
|
||||
NULL = 6,
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// LSM-Tree
|
||||
// =============================================================================
|
||||
|
||||
/** MemTable 最大大小（默认 4MB） */
|
||||
export const DEFAULT_MEMTABLE_SIZE = 4 * 1024 * 1024;
|
||||
|
||||
/** SSTable 中每个 Data Block 的默认大小 */
|
||||
export const DEFAULT_BLOCK_SIZE = 4096;
|
||||
|
||||
/** Bloom Filter 每 key 的默认位数 */
|
||||
export const DEFAULT_BLOOM_BITS_PER_KEY = 10;
|
||||
|
||||
/** SSTable 最大层级 */
|
||||
export const MAX_LSM_LEVELS = 7;
|
||||
|
||||
/** 每层之间的大小倍数 */
|
||||
export const DEFAULT_LEVEL_SIZE_MULTIPLIER = 10;
|
||||
|
||||
/** SSTable 元数据 */
|
||||
export interface SSTableMeta {
|
||||
/** SSTable 文件 ID */
|
||||
id: number;
|
||||
/** 所在层级 */
|
||||
level: number;
|
||||
/** 最小 key */
|
||||
minKey: string;
|
||||
/** 最大 key */
|
||||
maxKey: string;
|
||||
/** 数据块数量 */
|
||||
blockCount: number;
|
||||
/** 总大小（字节） */
|
||||
totalSize: number;
|
||||
/** Bloom Filter 序列化数据 */
|
||||
bloomData: Uint8Array | null;
|
||||
}
|
||||
|
||||
/** SSTable 内部的 Data Block */
|
||||
export interface DataBlock {
|
||||
/** 该块内的 key-value 条目数 */
|
||||
entryCount: number;
|
||||
/** 该块数据区 */
|
||||
data: Uint8Array;
|
||||
/** 该块起始 key */
|
||||
startKey: string;
|
||||
/** 该块结束 key */
|
||||
endKey: string;
|
||||
}
|
||||
|
||||
/** 索引块条目：key → data block offset */
|
||||
export interface IndexEntry {
|
||||
/** 到此 block 的最后一个 key */
|
||||
key: string;
|
||||
/** data block 在 SSTable 文件中的偏移 */
|
||||
blockOffset: number;
|
||||
/** data block 大小 */
|
||||
blockSize: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// WAL (Write-Ahead Log)
|
||||
// =============================================================================
|
||||
|
||||
/** WAL 记录类型 */
|
||||
export enum WALRecordType {
|
||||
INSERT = 1,
|
||||
UPDATE = 2,
|
||||
DELETE = 3,
|
||||
BEGIN = 4,
|
||||
COMMIT = 5,
|
||||
ROLLBACK = 6,
|
||||
CREATE_TABLE = 7,
|
||||
DROP_TABLE = 8,
|
||||
}
|
||||
|
||||
/** 单条 WAL 记录 */
|
||||
export interface WALRecord {
|
||||
/** 日志序列号 */
|
||||
lsn: number;
|
||||
/** 记录类型 */
|
||||
type: WALRecordType;
|
||||
/** 事务 ID */
|
||||
txnId: number;
|
||||
/** 表名 */
|
||||
tableName: string;
|
||||
/** 主键值 */
|
||||
key: string;
|
||||
/** 操作数据（INSERT/UPDATE 时有效） */
|
||||
data?: Record<string, unknown>;
|
||||
/** 校验和 */
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// MVCC
|
||||
// =============================================================================
|
||||
|
||||
/** 事务隔离级别 */
|
||||
export enum IsolationLevel {
|
||||
READ_COMMITTED = 1,
|
||||
SNAPSHOT = 2,
|
||||
}
|
||||
|
||||
/** 事务状态 */
|
||||
export enum TransactionState {
|
||||
ACTIVE = 1,
|
||||
COMMITTED = 2,
|
||||
ABORTED = 3,
|
||||
}
|
||||
|
||||
/** 行版本 */
|
||||
export interface RowVersion {
|
||||
/** 事务 ID（创建此版本的事务） */
|
||||
txnId: number;
|
||||
/** 版本数据 */
|
||||
data: Record<string, unknown>;
|
||||
/** 指向上一版本的指针（undo 链） */
|
||||
prevVersion: RowVersion | null;
|
||||
/** 该版本是否已提交 */
|
||||
committed: boolean;
|
||||
}
|
||||
|
||||
/** 活跃事务表项 */
|
||||
export interface TxnEntry {
|
||||
/** 事务 ID */
|
||||
txnId: number;
|
||||
/** 事务状态 */
|
||||
state: TransactionState;
|
||||
/** 快照序列号（用于可见性判断） */
|
||||
snapshotLsn: number;
|
||||
/** 事务开始时间 */
|
||||
startTime: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Buffer Pool
|
||||
// =============================================================================
|
||||
|
||||
/** Buffer Pool 默认容量：256 页 ≈ 1MB */
|
||||
export const DEFAULT_BUFFER_POOL_PAGES = 256;
|
||||
|
||||
// =============================================================================
|
||||
// AriaEngine 配置
|
||||
// =============================================================================
|
||||
|
||||
export interface AriaEngineConfig {
|
||||
/** 页面大小（默认 4096） */
|
||||
pageSize?: number;
|
||||
/** Buffer Pool 页面数量（默认 256） */
|
||||
bufferPoolPages?: number;
|
||||
/** MemTable 刷盘阈值（默认 4MB） */
|
||||
memtableSizeThreshold?: number;
|
||||
/** LSM 层级之间的容量倍数（默认 10） */
|
||||
levelSizeMultiplier?: number;
|
||||
/** Bloom Filter 每 key 位数（默认 10） */
|
||||
bloomFilterBitsPerKey?: number;
|
||||
/** 是否启用 WAL（默认 true） */
|
||||
walEnabled?: boolean;
|
||||
/** WAL 同步模式 */
|
||||
walSyncMode?: 'full' | 'batch' | 'none';
|
||||
/** Checkpoint 间隔（操作数，默认 1000） */
|
||||
checkpointInterval?: number;
|
||||
/** 是否启用页面压缩（默认 false） */
|
||||
compression?: boolean;
|
||||
/** 存储后端 */
|
||||
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
||||
/** WAL 大小阈值（字节，超过则强制 checkpoint，默认 16MB） */
|
||||
walSizeThreshold?: number;
|
||||
/** 最大内存预算（MB，默认 64） */
|
||||
maxMemoryMB?: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_ARIA_CONFIG: Required<AriaEngineConfig> = {
|
||||
pageSize: PAGE_SIZE,
|
||||
bufferPoolPages: DEFAULT_BUFFER_POOL_PAGES,
|
||||
memtableSizeThreshold: DEFAULT_MEMTABLE_SIZE,
|
||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||
walEnabled: true,
|
||||
walSyncMode: 'full',
|
||||
checkpointInterval: 1000,
|
||||
compression: false,
|
||||
storageBackend: 'indexeddb',
|
||||
walSizeThreshold: 16 * 1024 * 1024, // 16MB
|
||||
maxMemoryMB: 64,
|
||||
};
|
||||
/**
|
||||
* AriaEngine Types — 内部类型定义
|
||||
* @module engine/aria/types
|
||||
*
|
||||
* 页面式存储引擎的所有内部枚举、接口和常量。
|
||||
*/
|
||||
|
||||
// =============================================================================
|
||||
// 页面常量
|
||||
// =============================================================================
|
||||
|
||||
/** 页面大小：4KB */
|
||||
export const PAGE_SIZE = 4096;
|
||||
|
||||
/** 页面头大小：16 字节 */
|
||||
export const PAGE_HEADER_SIZE = 16;
|
||||
|
||||
// =============================================================================
|
||||
// 页面类型
|
||||
// =============================================================================
|
||||
|
||||
export enum PageType {
|
||||
/** 数据页面：存储 SSTable 字节切片 */
|
||||
DATA = 1,
|
||||
/** 索引页面：存储索引节点 */
|
||||
INDEX = 2,
|
||||
/** 溢出页面：存储大字段 */
|
||||
OVERFLOW = 3,
|
||||
/** 元数据页面：存储表/库元信息 */
|
||||
META = 4,
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 页面句柄（Buffer Pool 中的页面）
|
||||
// =============================================================================
|
||||
|
||||
export interface PageHandle {
|
||||
/** 页面 ID */
|
||||
pageId: number;
|
||||
/** 页面类型 */
|
||||
type: PageType;
|
||||
/** 页面数据缓冲区（4KB ArrayBuffer） */
|
||||
data: ArrayBuffer;
|
||||
/** 是否被修改（脏页） */
|
||||
dirty: boolean;
|
||||
/** 引用计数（pin count） */
|
||||
pins: number;
|
||||
/** LRU 链表前驱 */
|
||||
prev: PageHandle | null;
|
||||
/** LRU 链表后继 */
|
||||
next: PageHandle | null;
|
||||
/** 最后访问时间戳 */
|
||||
lastAccess: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// 列类型（内部二进制编码用）
|
||||
// =============================================================================
|
||||
|
||||
// =============================================================================
|
||||
// LSM-Tree
|
||||
// =============================================================================
|
||||
|
||||
/** MemTable 最大大小（默认 4MB） */
|
||||
export const DEFAULT_MEMTABLE_SIZE = 4 * 1024 * 1024;
|
||||
|
||||
/** Bloom Filter 每 key 的默认位数 */
|
||||
export const DEFAULT_BLOOM_BITS_PER_KEY = 10;
|
||||
|
||||
/** SSTable 最大层级 */
|
||||
export const MAX_LSM_LEVELS = 7;
|
||||
|
||||
/** 每层之间的大小倍数 */
|
||||
export const DEFAULT_LEVEL_SIZE_MULTIPLIER = 10;
|
||||
|
||||
/** SSTable 元数据 */
|
||||
export interface SSTableMeta {
|
||||
/** SSTable 文件 ID */
|
||||
id: number;
|
||||
/** 所在层级 */
|
||||
level: number;
|
||||
/** 最小 key */
|
||||
minKey: string;
|
||||
/** 最大 key */
|
||||
maxKey: string;
|
||||
/** 数据块数量 */
|
||||
blockCount: number;
|
||||
/** 总大小（字节） */
|
||||
totalSize: number;
|
||||
/** Bloom Filter 序列化数据 */
|
||||
bloomData: Uint8Array | null;
|
||||
/**
|
||||
* v0.4.5: 页面化存储（OPFS 后端）— SSTable 的物理页面 ID 列表。
|
||||
* 存在 → 数据以 4KB 页面存储（BufferPool/FileManager 管理）；
|
||||
* 缺失 → 整 value 存储（旧格式，兼容读取）。
|
||||
*/
|
||||
pageIds?: number[];
|
||||
}
|
||||
|
||||
/** 索引块条目：key → data block offset */
|
||||
export interface IndexEntry {
|
||||
/** 到此 block 的最后一个 key */
|
||||
key: string;
|
||||
/** data block 在 SSTable 文件中的偏移 */
|
||||
blockOffset: number;
|
||||
/** data block 大小 */
|
||||
blockSize: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// WAL (Write-Ahead Log)
|
||||
// =============================================================================
|
||||
|
||||
/** WAL 记录类型 */
|
||||
export enum WALRecordType {
|
||||
INSERT = 1,
|
||||
UPDATE = 2,
|
||||
DELETE = 3,
|
||||
BEGIN = 4,
|
||||
COMMIT = 5,
|
||||
ROLLBACK = 6,
|
||||
CREATE_TABLE = 7,
|
||||
DROP_TABLE = 8,
|
||||
}
|
||||
|
||||
/** 单条 WAL 记录 */
|
||||
export interface WALRecord {
|
||||
/** 日志序列号 */
|
||||
lsn: number;
|
||||
/** 记录类型 */
|
||||
type: WALRecordType;
|
||||
/** 事务 ID */
|
||||
txnId: number;
|
||||
/** 表名 */
|
||||
tableName: string;
|
||||
/** 主键值 */
|
||||
key: string;
|
||||
/** 操作数据（INSERT/UPDATE 时有效） */
|
||||
data?: Record<string, unknown>;
|
||||
/** 校验和 */
|
||||
checksum: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// MVCC
|
||||
// =============================================================================
|
||||
|
||||
/** 事务状态 */
|
||||
export enum TransactionState {
|
||||
ACTIVE = 1,
|
||||
COMMITTED = 2,
|
||||
ABORTED = 3,
|
||||
}
|
||||
|
||||
/** 行版本 */
|
||||
export interface RowVersion {
|
||||
/** 事务 ID（创建此版本的事务） */
|
||||
txnId: number;
|
||||
/** 版本数据 */
|
||||
data: Record<string, unknown>;
|
||||
/** 指向上一版本的指针（undo 链） */
|
||||
prevVersion: RowVersion | null;
|
||||
/** 该版本是否已提交 */
|
||||
committed: boolean;
|
||||
}
|
||||
|
||||
/** 活跃事务表项 */
|
||||
export interface TxnEntry {
|
||||
/** 事务 ID */
|
||||
txnId: number;
|
||||
/** 事务状态 */
|
||||
state: TransactionState;
|
||||
/** 快照序列号（用于可见性判断） */
|
||||
snapshotLsn: number;
|
||||
/** 事务开始时间 */
|
||||
startTime: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Buffer Pool
|
||||
// =============================================================================
|
||||
|
||||
/** Buffer Pool 默认容量：256 页 ≈ 1MB */
|
||||
export const DEFAULT_BUFFER_POOL_PAGES = 256;
|
||||
|
||||
// =============================================================================
|
||||
// AriaEngine 配置
|
||||
// =============================================================================
|
||||
|
||||
export interface AriaEngineConfig {
|
||||
/** 页面大小（默认 4096） */
|
||||
pageSize?: number;
|
||||
/** Buffer Pool 页面数量（默认 256） */
|
||||
bufferPoolPages?: number;
|
||||
/** MemTable 刷盘阈值（默认 4MB） */
|
||||
memtableSizeThreshold?: number;
|
||||
/** LSM 层级之间的容量倍数（默认 10） */
|
||||
levelSizeMultiplier?: number;
|
||||
/** Bloom Filter 每 key 位数（默认 10） */
|
||||
bloomFilterBitsPerKey?: number;
|
||||
/** 是否启用 WAL（默认 true） */
|
||||
walEnabled?: boolean;
|
||||
/** WAL 同步模式 */
|
||||
walSyncMode?: 'full' | 'batch' | 'none';
|
||||
/** Checkpoint 间隔（操作数，默认 1000） */
|
||||
checkpointInterval?: number;
|
||||
/** 是否启用页面压缩（默认 false） */
|
||||
compression?: boolean;
|
||||
/** 存储后端 */
|
||||
storageBackend?: 'indexeddb' | 'opfs' | 'memory';
|
||||
/** WAL 大小阈值（字节，超过则强制 checkpoint，默认 16MB） */
|
||||
walSizeThreshold?: number;
|
||||
/** 最大内存预算（MB，默认 64） */
|
||||
maxMemoryMB?: number;
|
||||
/**
|
||||
* v0.4.5: 全库 AES-256-GCM 加密（backend 层透明加解密，WAL/SSTable/Schema/元数据全覆盖）。
|
||||
* 密钥由 PBKDF2（salt 持久化于库内 __aria_keymeta）派生，重启用同一密码即可解密。
|
||||
*/
|
||||
encryption?: {
|
||||
/** 加密密码 */
|
||||
password: string;
|
||||
};
|
||||
/**
|
||||
* v0.4.5: SSTable 页面化存储（4KB 页面 + BufferPool/FileManager 管理，LRU 缓存）。
|
||||
* 默认：storageBackend === 'opfs' 时自动启用（大文件随机读/写放大优化）；
|
||||
* 显式 false 强制关闭（整 value 存储，兼容旧行为）。
|
||||
*/
|
||||
pageStorage?: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_ARIA_CONFIG: Required<Omit<AriaEngineConfig, 'encryption' | 'pageStorage'>> & {
|
||||
encryption: undefined;
|
||||
pageStorage: undefined;
|
||||
} = {
|
||||
pageSize: PAGE_SIZE,
|
||||
bufferPoolPages: DEFAULT_BUFFER_POOL_PAGES,
|
||||
memtableSizeThreshold: DEFAULT_MEMTABLE_SIZE,
|
||||
levelSizeMultiplier: DEFAULT_LEVEL_SIZE_MULTIPLIER,
|
||||
bloomFilterBitsPerKey: DEFAULT_BLOOM_BITS_PER_KEY,
|
||||
walEnabled: true,
|
||||
walSyncMode: 'full',
|
||||
checkpointInterval: 1000,
|
||||
compression: false,
|
||||
storageBackend: 'indexeddb',
|
||||
walSizeThreshold: 16 * 1024 * 1024, // 16MB
|
||||
maxMemoryMB: 64,
|
||||
encryption: undefined,
|
||||
pageStorage: undefined,
|
||||
};
|
||||
|
||||
@@ -1,82 +1,71 @@
|
||||
/**
|
||||
* AriaEngine Checkpoint — 检查点机制
|
||||
* @module engine/aria/wal/checkpoint
|
||||
*/
|
||||
|
||||
import type { LSM } from '../index/lsm';
|
||||
import type { WAL } from './log';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 简化的 flush 接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface Flushable {
|
||||
flushAll(): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CheckpointManager
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class CheckpointManager {
|
||||
private lsm: LSM;
|
||||
private wal: WAL;
|
||||
private flushable: Flushable | null;
|
||||
private interval: number;
|
||||
private opCount = 0;
|
||||
private walSizeThreshold: number;
|
||||
|
||||
constructor(
|
||||
lsm: LSM,
|
||||
wal: WAL,
|
||||
flushable: Flushable | null = null,
|
||||
interval: number = 1000,
|
||||
walSizeThreshold: number = 16 * 1024 * 1024,
|
||||
) {
|
||||
this.lsm = lsm;
|
||||
this.wal = wal;
|
||||
this.flushable = flushable;
|
||||
this.interval = interval;
|
||||
this.walSizeThreshold = walSizeThreshold;
|
||||
}
|
||||
|
||||
async tick(): Promise<void> {
|
||||
this.opCount++;
|
||||
// 检查操作计数或 WAL 大小是否超阈值
|
||||
if (this.opCount >= this.interval || this.getWALEstimatedSize() >= this.walSizeThreshold) {
|
||||
await this.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
/** 估算 WAL 大小（优先真实字节数，回退到缓冲计数估算） */
|
||||
private getWALEstimatedSize(): number {
|
||||
const wal = this.wal as unknown as { getBufferedBytes?: () => number; getBufferedCount?: () => number };
|
||||
if (typeof wal.getBufferedBytes === 'function') {
|
||||
const bytes = wal.getBufferedBytes();
|
||||
if (bytes > 0) return bytes;
|
||||
}
|
||||
const count = typeof wal.getBufferedCount === 'function' ? wal.getBufferedCount() : 0;
|
||||
return count * 200;
|
||||
}
|
||||
|
||||
async checkpoint(): Promise<void> {
|
||||
await this.lsm.flush();
|
||||
if (this.flushable) {
|
||||
await this.flushable.flushAll();
|
||||
}
|
||||
await this.wal.checkpoint();
|
||||
this.opCount = 0;
|
||||
}
|
||||
|
||||
async forceCheckpoint(): Promise<void> {
|
||||
await this.checkpoint();
|
||||
}
|
||||
|
||||
setInterval(ops: number): void {
|
||||
this.interval = ops;
|
||||
}
|
||||
|
||||
getOpCount(): number {
|
||||
return this.opCount;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine Checkpoint — 检查点机制
|
||||
* @module engine/aria/wal/checkpoint
|
||||
*/
|
||||
|
||||
import type { LSM } from '../index/lsm';
|
||||
import type { WAL } from './log';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 简化的 flush 接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface Flushable {
|
||||
flushAll(): Promise<void>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CheckpointManager
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class CheckpointManager {
|
||||
private lsm: LSM;
|
||||
private wal: WAL;
|
||||
private flushable: Flushable | null;
|
||||
private interval: number;
|
||||
private opCount = 0;
|
||||
private walSizeThreshold: number;
|
||||
|
||||
constructor(
|
||||
lsm: LSM,
|
||||
wal: WAL,
|
||||
flushable: Flushable | null = null,
|
||||
interval: number = 1000,
|
||||
walSizeThreshold: number = 16 * 1024 * 1024,
|
||||
) {
|
||||
this.lsm = lsm;
|
||||
this.wal = wal;
|
||||
this.flushable = flushable;
|
||||
this.interval = interval;
|
||||
this.walSizeThreshold = walSizeThreshold;
|
||||
}
|
||||
|
||||
async tick(): Promise<void> {
|
||||
this.opCount++;
|
||||
// 检查操作计数或 WAL 大小是否超阈值
|
||||
if (this.opCount >= this.interval || this.getWALEstimatedSize() >= this.walSizeThreshold) {
|
||||
await this.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
/** 估算 WAL 大小（优先真实字节数，回退到缓冲计数估算） */
|
||||
private getWALEstimatedSize(): number {
|
||||
const wal = this.wal as unknown as { getBufferedBytes?: () => number; getBufferedCount?: () => number };
|
||||
if (typeof wal.getBufferedBytes === 'function') {
|
||||
const bytes = wal.getBufferedBytes();
|
||||
if (bytes > 0) return bytes;
|
||||
}
|
||||
const count = typeof wal.getBufferedCount === 'function' ? wal.getBufferedCount() : 0;
|
||||
return count * 200;
|
||||
}
|
||||
|
||||
async checkpoint(): Promise<void> {
|
||||
await this.lsm.flush();
|
||||
if (this.flushable) {
|
||||
await this.flushable.flushAll();
|
||||
}
|
||||
await this.wal.checkpoint();
|
||||
this.opCount = 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+322
-325
@@ -1,325 +1,322 @@
|
||||
/**
|
||||
* AriaEngine WAL — Write-Ahead Log
|
||||
* @module engine/aria/wal/log
|
||||
*
|
||||
* 崩溃恢复前的写操作持久化日志。
|
||||
*
|
||||
* WAL 文件格式:
|
||||
* ┌──────────┬──────────────┬──────────┐
|
||||
* │ Record 1│ Record 2 │ ... │
|
||||
* │ 4B LSN │ │ │
|
||||
* │ 1B type │ │ │
|
||||
* │ 4B txnId│ │ │
|
||||
* │ 2B tblLen│ │ │
|
||||
* │ N table│ │ │
|
||||
* │ 2B keyLen│ │ │
|
||||
* │ N key │ │ │
|
||||
* │ 4B jsonLen│ │ │
|
||||
* │ N json │ │ │
|
||||
* │ 4B CRC │ │ │
|
||||
* └──────────┴──────────────┴──────────┘
|
||||
*/
|
||||
|
||||
import { WALRecordType, type WALRecord } from '../types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL 存储接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WALStore {
|
||||
/** 追加 WAL 记录 */
|
||||
append(data: Uint8Array): Promise<void>;
|
||||
/** 读取所有 WAL 记录 */
|
||||
readAll(): Promise<Uint8Array>;
|
||||
/** 截断 WAL（checkpoint 后清理） */
|
||||
truncate(): Promise<void>;
|
||||
/** 检查 WAL 是否存在 */
|
||||
exists(): Promise<boolean>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class WAL {
|
||||
private lsn = 0;
|
||||
private store: WALStore;
|
||||
private enabled: boolean;
|
||||
private buffer: Uint8Array[] = [];
|
||||
private syncMode: 'full' | 'batch' | 'none';
|
||||
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数（full/batch/none 通用） */
|
||||
private bufferedBytes = 0;
|
||||
|
||||
constructor(store: WALStore, enabled: boolean = true, syncMode: 'full' | 'batch' | 'none' = 'batch') {
|
||||
this.store = store;
|
||||
this.enabled = enabled;
|
||||
this.syncMode = syncMode;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 写入
|
||||
// =======================================================================
|
||||
|
||||
/** 追加一条 WAL 记录（full 模式同步等待写入完成） */
|
||||
async append(record: Omit<WALRecord, 'lsn' | 'checksum'>): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
|
||||
this.lsn++;
|
||||
const fullRecord: WALRecord = {
|
||||
...record,
|
||||
lsn: this.lsn,
|
||||
checksum: 0, // 稍后计算
|
||||
};
|
||||
|
||||
const bytes = this.encodeRecord(fullRecord);
|
||||
|
||||
if (this.syncMode === 'full') {
|
||||
try {
|
||||
await this.store.append(bytes);
|
||||
this.bufferedBytes += bytes.byteLength;
|
||||
} catch {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn('[AriaEngine WAL] Failed to append record');
|
||||
}
|
||||
} else if (this.syncMode === 'batch') {
|
||||
this.buffer.push(bytes);
|
||||
this.bufferedBytes += bytes.byteLength;
|
||||
}
|
||||
// 'none' mode: 不写 WAL
|
||||
}
|
||||
|
||||
/** 批量追加多条 WAL 记录（组提交：合并为一次底层写入，v0.3.1） */
|
||||
async appendBatch(records: Omit<WALRecord, 'lsn' | 'checksum'>[]): Promise<void> {
|
||||
if (!this.enabled || records.length === 0) return;
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
for (const record of records) {
|
||||
this.lsn++;
|
||||
chunks.push(this.encodeRecord({ ...record, lsn: this.lsn, checksum: 0 }));
|
||||
}
|
||||
const combined = this.mergeChunks(chunks);
|
||||
|
||||
if (this.syncMode === 'full') {
|
||||
try {
|
||||
await this.store.append(combined);
|
||||
this.bufferedBytes += combined.byteLength;
|
||||
} catch {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn('[AriaEngine WAL] Failed to append batch record');
|
||||
}
|
||||
} else if (this.syncMode === 'batch') {
|
||||
this.buffer.push(combined);
|
||||
this.bufferedBytes += combined.byteLength;
|
||||
}
|
||||
// 'none' mode: 不写 WAL
|
||||
}
|
||||
|
||||
/** 批量刷新缓冲的 WAL 记录 */
|
||||
async flush(): Promise<void> {
|
||||
if (!this.enabled || this.buffer.length === 0) return;
|
||||
|
||||
const combined = this.mergeChunks(this.buffer);
|
||||
await this.store.append(combined);
|
||||
this.buffer = [];
|
||||
}
|
||||
|
||||
/** 合并多个字节块为一个连续缓冲区 */
|
||||
private mergeChunks(chunks: Uint8Array[]): Uint8Array {
|
||||
if (chunks.length === 1) return chunks[0];
|
||||
const totalLen = chunks.reduce((sum, b) => sum + b.byteLength, 0);
|
||||
const combined = new Uint8Array(totalLen);
|
||||
let offset = 0;
|
||||
for (const buf of chunks) {
|
||||
combined.set(buf, offset);
|
||||
offset += buf.byteLength;
|
||||
}
|
||||
return combined;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 恢复
|
||||
// =======================================================================
|
||||
|
||||
/** 从 WAL 恢复未提交的事务数据 */
|
||||
async recover(
|
||||
applyRecord: (record: WALRecord) => void,
|
||||
): Promise<number> {
|
||||
if (!this.enabled) return 0;
|
||||
|
||||
const exists = await this.store.exists();
|
||||
if (!exists) return 0;
|
||||
|
||||
const data = await this.store.readAll();
|
||||
if (data.byteLength === 0) return 0;
|
||||
|
||||
const records = this.decodeAllRecords(data);
|
||||
for (const record of records) {
|
||||
applyRecord(record);
|
||||
}
|
||||
|
||||
this.lsn = records.length > 0 ? records[records.length - 1].lsn : 0;
|
||||
return records.length;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// Checkpoint
|
||||
// =======================================================================
|
||||
|
||||
/** Checkpoint 后清空 WAL */
|
||||
async checkpoint(): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
await this.flush();
|
||||
await this.store.truncate();
|
||||
this.lsn = 0;
|
||||
this.bufferedBytes = 0;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 统计
|
||||
// =======================================================================
|
||||
|
||||
isEnabled(): boolean {
|
||||
return this.enabled;
|
||||
}
|
||||
|
||||
getLSN(): number {
|
||||
return this.lsn;
|
||||
}
|
||||
|
||||
getBufferedCount(): number {
|
||||
return this.buffer.length;
|
||||
}
|
||||
|
||||
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数（full/batch/none 通用） */
|
||||
getBufferedBytes(): number {
|
||||
return this.bufferedBytes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 编解码
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private encodeRecord(record: WALRecord): Uint8Array {
|
||||
const encoder = new TextEncoder();
|
||||
const tableBytes = encoder.encode(record.tableName);
|
||||
const keyBytes = encoder.encode(record.key);
|
||||
const jsonStr = record.data ? JSON.stringify(record.data) : '';
|
||||
const jsonBytes = encoder.encode(jsonStr);
|
||||
|
||||
const size =
|
||||
4 + // LSN
|
||||
1 + // type
|
||||
4 + // txnId
|
||||
2 + tableBytes.length + // table
|
||||
2 + keyBytes.length + // key
|
||||
4 + jsonBytes.length + // json
|
||||
4; // CRC
|
||||
|
||||
const buf = new ArrayBuffer(size);
|
||||
const view = new DataView(buf);
|
||||
let offset = 0;
|
||||
|
||||
view.setUint32(offset, record.lsn, false);
|
||||
offset += 4;
|
||||
view.setUint8(offset, record.type);
|
||||
offset += 1;
|
||||
view.setUint32(offset, record.txnId, false);
|
||||
offset += 4;
|
||||
|
||||
view.setUint16(offset, tableBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(buf).set(tableBytes, offset);
|
||||
offset += tableBytes.length;
|
||||
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(buf).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
|
||||
view.setUint32(offset, jsonBytes.length, false);
|
||||
offset += 4;
|
||||
new Uint8Array(buf).set(jsonBytes, offset);
|
||||
offset += jsonBytes.length;
|
||||
|
||||
// 简单 CRC
|
||||
let crc = 0;
|
||||
const u8 = new Uint8Array(buf, 0, offset);
|
||||
for (let i = 0; i < u8.length; i++) {
|
||||
crc = ((crc << 5) - crc + u8[i]) | 0;
|
||||
}
|
||||
view.setUint32(offset, crc >>> 0, false);
|
||||
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
|
||||
private decodeAllRecords(data: Uint8Array): WALRecord[] {
|
||||
const records: WALRecord[] = [];
|
||||
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
while (offset + 15 <= data.byteLength) {
|
||||
try {
|
||||
const recordStart = offset;
|
||||
const lsn = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
const type = view.getUint8(offset) as WALRecordType;
|
||||
offset += 1;
|
||||
const txnId = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
|
||||
const tableLen = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
if (offset + tableLen > data.byteLength) break;
|
||||
const tableName = new TextDecoder().decode(data.slice(offset, offset + tableLen));
|
||||
offset += tableLen;
|
||||
|
||||
const keyLen = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
if (offset + keyLen > data.byteLength) break;
|
||||
const key = new TextDecoder().decode(data.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
|
||||
const jsonLen = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
if (offset + jsonLen > data.byteLength) break;
|
||||
let recordData: Record<string, unknown> | undefined;
|
||||
if (jsonLen > 0) {
|
||||
const json = new TextDecoder().decode(data.slice(offset, offset + jsonLen));
|
||||
try {
|
||||
recordData = JSON.parse(json);
|
||||
} catch { /* ok */ }
|
||||
}
|
||||
offset += jsonLen;
|
||||
|
||||
// 验证 CRC（跨记录数据计算，不含 CRC 自身）
|
||||
const storedCrc = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
const recordBytes = data.slice(recordStart, offset - 4);
|
||||
let computedCrc = 0;
|
||||
for (let i = 0; i < recordBytes.length; i++) {
|
||||
computedCrc = ((computedCrc << 5) - computedCrc + recordBytes[i]) | 0;
|
||||
}
|
||||
if ((computedCrc >>> 0) !== storedCrc) {
|
||||
// CRC 不匹配，跳过此损坏记录
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`[AriaEngine WAL] CRC mismatch at record LSN=${lsn}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
records.push({
|
||||
lsn,
|
||||
type,
|
||||
txnId,
|
||||
tableName,
|
||||
key,
|
||||
data: recordData,
|
||||
checksum: storedCrc,
|
||||
});
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine WAL — Write-Ahead Log
|
||||
* @module engine/aria/wal/log
|
||||
*
|
||||
* 崩溃恢复前的写操作持久化日志。
|
||||
*
|
||||
* WAL 文件格式:
|
||||
* ┌──────────┬──────────────┬──────────┐
|
||||
* │ Record 1│ Record 2 │ ... │
|
||||
* │ 4B LSN │ │ │
|
||||
* │ 1B type │ │ │
|
||||
* │ 4B txnId│ │ │
|
||||
* │ 2B tblLen│ │ │
|
||||
* │ N table│ │ │
|
||||
* │ 2B keyLen│ │ │
|
||||
* │ N key │ │ │
|
||||
* │ 4B jsonLen│ │ │
|
||||
* │ N json │ │ │
|
||||
* │ 4B CRC │ │ │
|
||||
* └──────────┴──────────────┴──────────┘
|
||||
*/
|
||||
|
||||
import { WALRecordType, type WALRecord } from '../types';
|
||||
import { crc32 } from '../crc32';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL 存储接口
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WALStore {
|
||||
/** 追加 WAL 记录 */
|
||||
append(data: Uint8Array): Promise<void>;
|
||||
/** 读取所有 WAL 记录 */
|
||||
readAll(): Promise<Uint8Array>;
|
||||
/** 截断 WAL（checkpoint 后清理） */
|
||||
truncate(): Promise<void>;
|
||||
/** 检查 WAL 是否存在 */
|
||||
exists(): Promise<boolean>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class WAL {
|
||||
private lsn = 0;
|
||||
private store: WALStore;
|
||||
private enabled: boolean;
|
||||
private buffer: Uint8Array[] = [];
|
||||
private syncMode: 'full' | 'batch' | 'none';
|
||||
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数（full/batch/none 通用） */
|
||||
private bufferedBytes = 0;
|
||||
|
||||
constructor(store: WALStore, enabled: boolean = true, syncMode: 'full' | 'batch' | 'none' = 'batch') {
|
||||
this.store = store;
|
||||
this.enabled = enabled;
|
||||
this.syncMode = syncMode;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 写入
|
||||
// =======================================================================
|
||||
|
||||
/** 追加一条 WAL 记录（full 模式同步等待写入完成） */
|
||||
async append(record: Omit<WALRecord, 'lsn' | 'checksum'>): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
|
||||
this.lsn++;
|
||||
const fullRecord: WALRecord = {
|
||||
...record,
|
||||
lsn: this.lsn,
|
||||
checksum: 0, // 稍后计算
|
||||
};
|
||||
|
||||
const bytes = this.encodeRecord(fullRecord);
|
||||
|
||||
if (this.syncMode === 'full') {
|
||||
try {
|
||||
await this.store.append(bytes);
|
||||
this.bufferedBytes += bytes.byteLength;
|
||||
} catch {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn('[AriaEngine WAL] Failed to append record');
|
||||
}
|
||||
} else if (this.syncMode === 'batch') {
|
||||
this.buffer.push(bytes);
|
||||
this.bufferedBytes += bytes.byteLength;
|
||||
}
|
||||
// 'none' mode: 不写 WAL
|
||||
}
|
||||
|
||||
/** 批量追加多条 WAL 记录（组提交：合并为一次底层写入，v0.3.1） */
|
||||
async appendBatch(records: Omit<WALRecord, 'lsn' | 'checksum'>[]): Promise<void> {
|
||||
if (!this.enabled || records.length === 0) return;
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
for (const record of records) {
|
||||
this.lsn++;
|
||||
chunks.push(this.encodeRecord({ ...record, lsn: this.lsn, checksum: 0 }));
|
||||
}
|
||||
const combined = this.mergeChunks(chunks);
|
||||
|
||||
if (this.syncMode === 'full') {
|
||||
try {
|
||||
await this.store.append(combined);
|
||||
this.bufferedBytes += combined.byteLength;
|
||||
} catch {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn('[AriaEngine WAL] Failed to append batch record');
|
||||
}
|
||||
} else if (this.syncMode === 'batch') {
|
||||
this.buffer.push(combined);
|
||||
this.bufferedBytes += combined.byteLength;
|
||||
}
|
||||
// 'none' mode: 不写 WAL
|
||||
}
|
||||
|
||||
/** 批量刷新缓冲的 WAL 记录 */
|
||||
async flush(): Promise<void> {
|
||||
if (!this.enabled || this.buffer.length === 0) return;
|
||||
|
||||
const combined = this.mergeChunks(this.buffer);
|
||||
await this.store.append(combined);
|
||||
this.buffer = [];
|
||||
}
|
||||
|
||||
/** 合并多个字节块为一个连续缓冲区 */
|
||||
private mergeChunks(chunks: Uint8Array[]): Uint8Array {
|
||||
if (chunks.length === 1) return chunks[0];
|
||||
const totalLen = chunks.reduce((sum, b) => sum + b.byteLength, 0);
|
||||
const combined = new Uint8Array(totalLen);
|
||||
let offset = 0;
|
||||
for (const buf of chunks) {
|
||||
combined.set(buf, offset);
|
||||
offset += buf.byteLength;
|
||||
}
|
||||
return combined;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 恢复
|
||||
// =======================================================================
|
||||
|
||||
/** 从 WAL 恢复未提交的事务数据 */
|
||||
async recover(
|
||||
applyRecord: (record: WALRecord) => void,
|
||||
): Promise<number> {
|
||||
if (!this.enabled) return 0;
|
||||
|
||||
const exists = await this.store.exists();
|
||||
if (!exists) return 0;
|
||||
|
||||
const data = await this.store.readAll();
|
||||
if (data.byteLength === 0) return 0;
|
||||
|
||||
const records = this.decodeAllRecords(data);
|
||||
for (const record of records) {
|
||||
applyRecord(record);
|
||||
}
|
||||
|
||||
this.lsn = records.length > 0 ? records[records.length - 1].lsn : 0;
|
||||
return records.length;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// Checkpoint
|
||||
// =======================================================================
|
||||
|
||||
/** Checkpoint 后清空 WAL */
|
||||
async checkpoint(): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
await this.flush();
|
||||
await this.store.truncate();
|
||||
this.lsn = 0;
|
||||
this.bufferedBytes = 0;
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 统计
|
||||
// =======================================================================
|
||||
|
||||
getBufferedCount(): number {
|
||||
return this.buffer.length;
|
||||
}
|
||||
|
||||
/** v0.3.3: 未 checkpoint 的 WAL 累计字节数（full/batch/none 通用） */
|
||||
getBufferedBytes(): number {
|
||||
return this.bufferedBytes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 编解码
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/** 旧版弱滚动校验（v0.4.4 及更早写入的 WAL 记录使用，双算法探测兼容） */
|
||||
private legacyChecksum(data: Uint8Array): number {
|
||||
let crc = 0;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
crc = ((crc << 5) - crc + data[i]) | 0;
|
||||
}
|
||||
return crc >>> 0;
|
||||
}
|
||||
|
||||
private encodeRecord(record: WALRecord): Uint8Array {
|
||||
const encoder = new TextEncoder();
|
||||
const tableBytes = encoder.encode(record.tableName);
|
||||
const keyBytes = encoder.encode(record.key);
|
||||
const jsonStr = record.data ? JSON.stringify(record.data) : '';
|
||||
const jsonBytes = encoder.encode(jsonStr);
|
||||
|
||||
const size =
|
||||
4 + // LSN
|
||||
1 + // type
|
||||
4 + // txnId
|
||||
2 + tableBytes.length + // table
|
||||
2 + keyBytes.length + // key
|
||||
4 + jsonBytes.length + // json
|
||||
4; // CRC
|
||||
|
||||
const buf = new ArrayBuffer(size);
|
||||
const view = new DataView(buf);
|
||||
let offset = 0;
|
||||
|
||||
view.setUint32(offset, record.lsn, false);
|
||||
offset += 4;
|
||||
view.setUint8(offset, record.type);
|
||||
offset += 1;
|
||||
view.setUint32(offset, record.txnId, false);
|
||||
offset += 4;
|
||||
|
||||
view.setUint16(offset, tableBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(buf).set(tableBytes, offset);
|
||||
offset += tableBytes.length;
|
||||
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(buf).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
|
||||
view.setUint32(offset, jsonBytes.length, false);
|
||||
offset += 4;
|
||||
new Uint8Array(buf).set(jsonBytes, offset);
|
||||
offset += jsonBytes.length;
|
||||
|
||||
// v0.4.5: 标准 CRC-32 校验（此前为弱滚动校验，误检率更高）
|
||||
const u8 = new Uint8Array(buf, 0, offset);
|
||||
const crc = crc32(u8);
|
||||
view.setUint32(offset, crc, false);
|
||||
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
|
||||
private decodeAllRecords(data: Uint8Array): WALRecord[] {
|
||||
const records: WALRecord[] = [];
|
||||
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
let offset = 0;
|
||||
|
||||
while (offset + 15 <= data.byteLength) {
|
||||
try {
|
||||
const recordStart = offset;
|
||||
const lsn = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
const type = view.getUint8(offset) as WALRecordType;
|
||||
offset += 1;
|
||||
const txnId = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
|
||||
const tableLen = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
if (offset + tableLen > data.byteLength) break;
|
||||
const tableName = new TextDecoder().decode(data.slice(offset, offset + tableLen));
|
||||
offset += tableLen;
|
||||
|
||||
const keyLen = view.getUint16(offset, false);
|
||||
offset += 2;
|
||||
if (offset + keyLen > data.byteLength) break;
|
||||
const key = new TextDecoder().decode(data.slice(offset, offset + keyLen));
|
||||
offset += keyLen;
|
||||
|
||||
const jsonLen = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
if (offset + jsonLen > data.byteLength) break;
|
||||
let recordData: Record<string, unknown> | undefined;
|
||||
if (jsonLen > 0) {
|
||||
const json = new TextDecoder().decode(data.slice(offset, offset + jsonLen));
|
||||
try {
|
||||
recordData = JSON.parse(json);
|
||||
} catch { /* ok */ }
|
||||
}
|
||||
offset += jsonLen;
|
||||
|
||||
// 验证 CRC：先标准 CRC-32，失败再尝试旧版弱滚动校验（兼容旧库 WAL 记录）
|
||||
const storedCrc = view.getUint32(offset, false);
|
||||
offset += 4;
|
||||
const recordBytes = data.slice(recordStart, offset - 4);
|
||||
const computedNew = crc32(recordBytes);
|
||||
const computedLegacy = this.legacyChecksum(recordBytes);
|
||||
if ((computedNew >>> 0) !== storedCrc && (computedLegacy >>> 0) !== storedCrc) {
|
||||
// CRC 不匹配，跳过此损坏记录
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`[AriaEngine WAL] CRC mismatch at record LSN=${lsn}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
records.push({
|
||||
lsn,
|
||||
type,
|
||||
txnId,
|
||||
tableName,
|
||||
key,
|
||||
data: recordData,
|
||||
checksum: storedCrc,
|
||||
});
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* AriaEngine Segmented WAL Store — 分片式 WAL 持久化存储
|
||||
* @module engine/aria/wal/segmented_store
|
||||
*
|
||||
* v0.4.5: 取代"每条记录一个 key"的旧结构：
|
||||
* - 分片文件 `__wal_%06d.bin`，达到阈值（默认 4MB）切新分片 → 文件数量可控
|
||||
* - append 追加写当前分片（后端支持真追加则 O(chunk)，否则回退 read+write）
|
||||
* - WAL 序号（LSN）内嵌于记录字节流，无需独立 count 键 → append 单文件原子写
|
||||
* - readAll 检测分片序号空洞：序号不连续 → 丢弃空洞之后的分片（保守截断，
|
||||
* 空洞仅可能来自 truncate 部分完成——此时数据已落盘，丢弃无害）
|
||||
* - 兼容旧格式 `__wal_N`（每条记录一个键）+ `__wal_count`：读取时迁移重放，
|
||||
* checkpoint 时一并清空
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from '../store/backend';
|
||||
|
||||
/** 分片文件名：__wal_%06d.bin */
|
||||
export const WAL_SEGMENT_PREFIX = '__wal_';
|
||||
const SEGMENT_REGEX = /^__wal_(\d{6})\.bin$/;
|
||||
const LEGACY_RECORD_REGEX = /^__wal_(\d+)$/;
|
||||
const LEGACY_COUNT_KEY = '__wal_count';
|
||||
|
||||
/** 默认分片阈值 */
|
||||
export const DEFAULT_WAL_SEGMENT_SIZE = 4 * 1024 * 1024;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAL 存储接口（与 WAL 类解耦）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WALStore {
|
||||
append(data: Uint8Array): Promise<void>;
|
||||
readAll(): Promise<Uint8Array>;
|
||||
truncate(): Promise<void>;
|
||||
exists(): Promise<boolean>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SegmentedWALStore
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class SegmentedWALStore implements WALStore {
|
||||
private segmentSize: number;
|
||||
/** 当前分片序号（append 定位） */
|
||||
private currentSegment = 0;
|
||||
/** 当前分片字节数（内存跟踪，append 切分片判断） */
|
||||
private currentSize = 0;
|
||||
|
||||
constructor(
|
||||
private backend: IStorageBackend,
|
||||
segmentSize: number = DEFAULT_WAL_SEGMENT_SIZE,
|
||||
) {
|
||||
this.segmentSize = segmentSize;
|
||||
}
|
||||
|
||||
/** 分片 key 生成 */
|
||||
private segmentKey(seq: number): string {
|
||||
return `${WAL_SEGMENT_PREFIX}${String(seq).padStart(6, '0')}.bin`;
|
||||
}
|
||||
|
||||
async append(data: Uint8Array): Promise<void> {
|
||||
if (data.byteLength === 0) return;
|
||||
if (this.currentSize + data.byteLength > this.segmentSize) {
|
||||
// 当前分片放不下 → 切新分片
|
||||
this.currentSegment++;
|
||||
this.currentSize = 0;
|
||||
}
|
||||
const key = this.segmentKey(this.currentSegment);
|
||||
const copy = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
|
||||
|
||||
if (typeof this.backend.append === 'function') {
|
||||
await this.backend.append!(key, copy);
|
||||
} else {
|
||||
// 回退：读旧 + 拼接 + 写（单文件原子写）
|
||||
const existing = await this.backend.read(key);
|
||||
if (existing) {
|
||||
const combined = new ArrayBuffer(existing.byteLength + copy.byteLength);
|
||||
new Uint8Array(combined).set(new Uint8Array(existing), 0);
|
||||
new Uint8Array(combined).set(new Uint8Array(copy), existing.byteLength);
|
||||
await this.backend.write(key, combined);
|
||||
} else {
|
||||
await this.backend.write(key, copy);
|
||||
}
|
||||
}
|
||||
this.currentSize += data.byteLength;
|
||||
}
|
||||
|
||||
async readAll(): Promise<Uint8Array> {
|
||||
const keys = await this.backend.listKeys();
|
||||
|
||||
// ---- 新格式分片 ----
|
||||
const segments = keys
|
||||
.filter((k) => SEGMENT_REGEX.test(k))
|
||||
.map((k) => ({ seq: Number(k.match(SEGMENT_REGEX)![1]), key: k }))
|
||||
.sort((a, b) => a.seq - b.seq);
|
||||
|
||||
// 空洞检测：分片序号必须从 0 严格连续，空洞后的分片整体丢弃
|
||||
let keepCount = 0;
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
if (segments[i].seq !== i) break;
|
||||
keepCount = i + 1;
|
||||
}
|
||||
const validSegments = segments.slice(0, keepCount);
|
||||
|
||||
// ---- 旧格式兼容：__wal_N 单记录键（迁移前数据） ----
|
||||
const legacyKeys = keys
|
||||
.filter((k) => LEGACY_RECORD_REGEX.test(k))
|
||||
.map((k) => ({ seq: Number(k.match(LEGACY_RECORD_REGEX)![1]), key: k }))
|
||||
.sort((a, b) => a.seq - b.seq);
|
||||
|
||||
const parts: Uint8Array[] = [];
|
||||
// 旧格式在前（它们是最早的记录）
|
||||
for (const { key } of legacyKeys) {
|
||||
const raw = await this.backend.read(key);
|
||||
if (raw) parts.push(new Uint8Array(raw));
|
||||
}
|
||||
// 新格式分片在后
|
||||
for (const { key } of validSegments) {
|
||||
const raw = await this.backend.read(key);
|
||||
if (raw) parts.push(new Uint8Array(raw));
|
||||
}
|
||||
|
||||
// 同步当前分片状态（追加定位）
|
||||
if (validSegments.length > 0) {
|
||||
this.currentSegment = validSegments[validSegments.length - 1].seq;
|
||||
const lastRaw = await this.backend.read(validSegments[validSegments.length - 1].key);
|
||||
this.currentSize = lastRaw ? lastRaw.byteLength : 0;
|
||||
// 旧格式键存在时（迁移中），下一条记录另起分片，避免与旧键序号冲突
|
||||
if (legacyKeys.length > 0) {
|
||||
this.currentSegment++;
|
||||
this.currentSize = 0;
|
||||
}
|
||||
} else if (legacyKeys.length > 0) {
|
||||
// 仅有旧格式：迁移中，新写入从分片 0 开始（checkpoint 会清空旧键）
|
||||
this.currentSegment = 0;
|
||||
this.currentSize = 0;
|
||||
}
|
||||
|
||||
const total = parts.reduce((s, c) => s + c.byteLength, 0);
|
||||
const combined = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of parts) { combined.set(c, off); off += c.byteLength; }
|
||||
return combined;
|
||||
}
|
||||
|
||||
async truncate(): Promise<void> {
|
||||
const keys = await this.backend.listKeys();
|
||||
const walKeys = keys.filter((k) =>
|
||||
k.startsWith(WAL_SEGMENT_PREFIX) || k === LEGACY_COUNT_KEY);
|
||||
if (walKeys.length > 0) {
|
||||
await this.backend.deleteMany(walKeys);
|
||||
}
|
||||
this.currentSegment = 0;
|
||||
this.currentSize = 0;
|
||||
}
|
||||
|
||||
async exists(): Promise<boolean> {
|
||||
const keys = await this.backend.listKeys();
|
||||
return keys.some((k) =>
|
||||
k.startsWith(WAL_SEGMENT_PREFIX) || k === LEGACY_COUNT_KEY);
|
||||
}
|
||||
}
|
||||
@@ -360,16 +360,4 @@ export class OPFSEngine implements IStorageEngine {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 从 OPFS 加载表数据到内存缓存 */
|
||||
async loadTableIntoMemory(tableName: string, schema: TableSchema): Promise<void> {
|
||||
await this.memoryCache.createTable(schema);
|
||||
const rows = await this.readTableData(tableName);
|
||||
if (rows.length > 0) {
|
||||
// 直接用 Map 设置绕过 insert 校验
|
||||
for (const row of rows) {
|
||||
await this.memoryCache.insert(tableName, [row]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user